From 9bf6b0baa35ff17440455fa99f3ad8301f145bee Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Thu, 9 Apr 2020 02:21:59 +0000
Subject: [PATCH 01/19] feat: interface declaration of waysnodes

---
 integration/util/waysnodes/interface.go | 31 +++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 integration/util/waysnodes/interface.go

diff --git a/integration/util/waysnodes/interface.go b/integration/util/waysnodes/interface.go
new file mode 100644
index 00000000000..fcefacbc909
--- /dev/null
+++ b/integration/util/waysnodes/interface.go
@@ -0,0 +1,31 @@
+// Package waysnodes is designed to map between wayIDs and nodeIDs, e.g.,
+//   `wayID->nodeIDs`
+//   `fromNodeID,toNodeID->wayID(directed)`, etc.
+package waysnodes
+
+// NodesQuerier is the interface that wraps the QueryNodes method.
+type NodesQuerier interface {
+
+	// QueryNodes queries nodeIDs by wayID.
+	// wayID: positive means travel forward following the nodes sequence, negative means backward
+	// returned nodeIDs: the sequence will be reversed if wayID is backward.
+	QueryNodes(wayID int64) (nodeIDs []int64, err error)
+}
+
+// WayQuerier is the interface that wraps the QueryWay method.
+type WayQuerier interface {
+
+	// QueryWay queries directed wayID by fromNodeID,toNodeID pair.
+	// returned wayID: positive means travel forward following the fromNodeID,toNodeID sequence, negative means backward
+	QueryWay(fromNodeID, toNodeID int64) (wayID int64, err error)
+}
+
+// WaysQuerier is the interface that wraps the QueryWays method.
+type WaysQuerier interface {
+	WayQuerier
+
+	// QueryWays queries directed wayIDs by nodeIDs.
+	// `len(wayIDs) == len(nodeIDs)-1` since each way have to be decided by traveling from one node to another.
+	// returned wayIDs: positive means travel forward following the nodeIDs sequence, negative means backward
+	QueryWays(nodeIDs []int64) (wayIDs []int64, err error)
+}

From ebd9e7d44843bc6d49563e92ccd8afd769a365c7 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Thu, 9 Apr 2020 03:36:06 +0000
Subject: [PATCH 02/19] feat: add writer interface

---
 .../util/waysnodes/{interface.go => query_interface.go}  | 0
 integration/util/waysnodes/update_interface.go           | 9 +++++++++
 2 files changed, 9 insertions(+)
 rename integration/util/waysnodes/{interface.go => query_interface.go} (100%)
 create mode 100644 integration/util/waysnodes/update_interface.go

diff --git a/integration/util/waysnodes/interface.go b/integration/util/waysnodes/query_interface.go
similarity index 100%
rename from integration/util/waysnodes/interface.go
rename to integration/util/waysnodes/query_interface.go
diff --git a/integration/util/waysnodes/update_interface.go b/integration/util/waysnodes/update_interface.go
new file mode 100644
index 00000000000..58305fb1dc8
--- /dev/null
+++ b/integration/util/waysnodes/update_interface.go
@@ -0,0 +1,9 @@
+package waysnodes
+
+// Writer is the interface that wraps the Write method.
+type Writer interface {
+
+	// Write writes wayID->nodeIDs mapping into cache or storage.
+	// wayID: is undirected when input, so will always be positive.
+	Write(wayID int64, nodeIDs int64) error
+}

From b9d6a6d77a2b376ea97ac74275bd3b5dfa79232a Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Thu, 9 Apr 2020 11:22:09 +0000
Subject: [PATCH 03/19] feat: implement nodes2way stores in boltdb

---
 .../util/waysnodes/nodes2wayblotdb/db.go      | 101 ++++++++++++++++++
 .../waysnodes/nodes2wayblotdb/key_value.go    |  44 ++++++++
 .../nodes2wayblotdb/key_value_test.go         |  53 +++++++++
 .../util/waysnodes/update_interface.go        |   4 +-
 4 files changed, 200 insertions(+), 2 deletions(-)
 create mode 100644 integration/util/waysnodes/nodes2wayblotdb/db.go
 create mode 100644 integration/util/waysnodes/nodes2wayblotdb/key_value.go
 create mode 100644 integration/util/waysnodes/nodes2wayblotdb/key_value_test.go

diff --git a/integration/util/waysnodes/nodes2wayblotdb/db.go b/integration/util/waysnodes/nodes2wayblotdb/db.go
new file mode 100644
index 00000000000..b132250ebbf
--- /dev/null
+++ b/integration/util/waysnodes/nodes2wayblotdb/db.go
@@ -0,0 +1,101 @@
+// Package nodes2wayblotdb stores `fromNodeID,toNodeID -> wayID` mapping in blotdb.
+package nodes2wayblotdb
+
+import (
+	"errors"
+	"fmt"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+// DB stores `fromNodeID,toNodeID -> wayID` in blotdb.
+type DB struct {
+	db *bolt.DB
+}
+
+const (
+	defaultBucket = "bucket"
+)
+
+var (
+	errEmptyDB = errors.New("empty db")
+)
+
+// Open creates/opens a DB structure to store the nodes2way mapping.
+func Open(dbFilePath string, readOnly bool) (*DB, error) {
+	var db DB
+
+	options := bolt.Options{
+
+		// Default Bolt Options
+		Timeout:      0,
+		NoGrowSync:   false,
+		FreelistType: bolt.FreelistArrayType,
+
+		// set readonly
+		ReadOnly: readOnly,
+	}
+
+	var err error
+	db.db, err = bolt.Open(dbFilePath, 0666, &options)
+	if err != nil {
+		return nil, err
+	}
+
+	if readOnly {
+		return &db, nil
+	}
+
+	// for write, make sure bucket available
+	err = db.db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(defaultBucket))
+		if err != nil {
+			return fmt.Errorf("failed to create bucket: %s", err)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &db, nil
+}
+
+// Close releases all database resources.
+func (db *DB) Close() error {
+	if db.db == nil {
+		return errEmptyDB
+	}
+
+	return db.db.Close()
+}
+
+// Write writes wayID and its nodeIDs into cache or storage.
+// wayID: is undirected when input, so will always be positive.
+func (db *DB) Write(wayID int64, nodeIDs []int64) error {
+	if db.db == nil {
+		return errEmptyDB
+	}
+	if wayID < 0 {
+		return fmt.Errorf("expect undirected wayID")
+	}
+	if len(nodeIDs) < 2 {
+		return fmt.Errorf("at least 2 nodes for a way")
+	}
+
+	err := db.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(defaultBucket))
+
+		for i := 0; i < len(nodeIDs)-1; i++ {
+			if err := b.Put(key(nodeIDs[i], nodeIDs[i+1]), value(wayID)); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/integration/util/waysnodes/nodes2wayblotdb/key_value.go b/integration/util/waysnodes/nodes2wayblotdb/key_value.go
new file mode 100644
index 00000000000..d9ca218357e
--- /dev/null
+++ b/integration/util/waysnodes/nodes2wayblotdb/key_value.go
@@ -0,0 +1,44 @@
+package nodes2wayblotdb
+
+import (
+	"encoding/binary"
+
+	"github.com/golang/glog"
+)
+
+const (
+	int64Bytes = 8
+
+	maxKeyBytes   = 2 * int64Bytes // 2 int64
+	maxValueBytes = int64Bytes     // 1 int64
+)
+
+func key(fromNodeID, toNodeID int64) []byte {
+
+	buf := make([]byte, maxKeyBytes, maxKeyBytes)
+	binary.LittleEndian.PutUint64(buf, uint64(fromNodeID))
+	binary.LittleEndian.PutUint64(buf[int64Bytes:], uint64(toNodeID))
+	return buf
+}
+
+func parseKey(buf []byte) (fromNodeID, toNodeID int64) {
+	if len(buf) < maxKeyBytes {
+		glog.Fatalf("invalid key buf: %v\n", buf)
+		return
+	}
+
+	fromNodeID = int64(binary.LittleEndian.Uint64(buf))
+	toNodeID = int64(binary.LittleEndian.Uint64(buf[int64Bytes:]))
+	return
+}
+
+func value(wayID int64) []byte {
+	buf := make([]byte, maxValueBytes, maxValueBytes)
+	binary.LittleEndian.PutUint64(buf, uint64(wayID))
+	return buf
+}
+
+func parseValue(buf []byte) (wayID int64) {
+	wayID = int64(binary.LittleEndian.Uint64(buf))
+	return
+}
diff --git a/integration/util/waysnodes/nodes2wayblotdb/key_value_test.go b/integration/util/waysnodes/nodes2wayblotdb/key_value_test.go
new file mode 100644
index 00000000000..2e454a66b5a
--- /dev/null
+++ b/integration/util/waysnodes/nodes2wayblotdb/key_value_test.go
@@ -0,0 +1,53 @@
+package nodes2wayblotdb
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestKey(t *testing.T) {
+	cases := []struct {
+		fromNodeID int64
+		toNodeID   int64
+		key        []byte
+	}{
+		{0, 0, []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
+		{24418325, 84760891102, []byte{0x15, 0x98, 0x74, 0x01, 0x00, 0x00, 0x00, 0x00, 0xDE, 0x8E, 0x24, 0xBC, 0x13, 0x00, 0x00, 0x00}},
+	}
+
+	for _, c := range cases {
+		k := key(c.fromNodeID, c.toNodeID)
+		if len(k) == 0 || !reflect.DeepEqual(k, c.key) {
+			t.Errorf("generate key from %d,%d, expect to have %v, but got %v", c.fromNodeID, c.toNodeID, c.key, k)
+		}
+
+		from, to := parseKey(c.key)
+		if from != c.fromNodeID || to != c.toNodeID {
+			t.Errorf("parse key from %v, expect %d,%d but got %d,%d", c.key, c.fromNodeID, c.toNodeID, from, to)
+		}
+	}
+
+}
+
+func TestValue(t *testing.T) {
+	cases := []struct {
+		wayID int64
+		value []byte
+	}{
+		{0, []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}},
+		{84760891102, []byte{0xDE, 0x8E, 0x24, 0xBC, 0x13, 0x00, 0x00, 0x00}},
+	}
+
+	for _, c := range cases {
+		v := value(c.wayID)
+		if len(v) == 0 || !reflect.DeepEqual(v, c.value) {
+			t.Errorf("generate value from %d, expect to have %v, but got %v", c.wayID, c.value, v)
+		}
+
+		wayID := parseValue(c.value)
+		if wayID != c.wayID {
+			t.Errorf("parse value from %v, expect %d but got %d", c.value, c.wayID, wayID)
+		}
+	}
+
+}
diff --git a/integration/util/waysnodes/update_interface.go b/integration/util/waysnodes/update_interface.go
index 58305fb1dc8..53ece671600 100644
--- a/integration/util/waysnodes/update_interface.go
+++ b/integration/util/waysnodes/update_interface.go
@@ -3,7 +3,7 @@ package waysnodes
 // Writer is the interface that wraps the Write method.
 type Writer interface {
 
-	// Write writes wayID->nodeIDs mapping into cache or storage.
+	// Write writes wayID and its nodeIDs into cache or storage.
 	// wayID: is undirected when input, so will always be positive.
-	Write(wayID int64, nodeIDs int64) error
+	Write(wayID int64, nodeIDs []int64) error
 }

From 1a6cbb48b3e920b36d1bce5154d00c05b42772d4 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Thu, 9 Apr 2020 12:04:23 +0000
Subject: [PATCH 04/19] feat: implement query way/ways

---
 .../util/waysnodes/nodes2wayblotdb/db.go      | 70 ++++++++++++++++++-
 1 file changed, 69 insertions(+), 1 deletion(-)

diff --git a/integration/util/waysnodes/nodes2wayblotdb/db.go b/integration/util/waysnodes/nodes2wayblotdb/db.go
index b132250ebbf..25eeb8c8fe3 100644
--- a/integration/util/waysnodes/nodes2wayblotdb/db.go
+++ b/integration/util/waysnodes/nodes2wayblotdb/db.go
@@ -18,7 +18,8 @@ const (
 )
 
 var (
-	errEmptyDB = errors.New("empty db")
+	errEmptyDB  = errors.New("empty db")
+	errNotFound = errors.New("not found")
 )
 
 // Open creates/opens a DB structure to store the nodes2way mapping.
@@ -99,3 +100,70 @@ func (db *DB) Write(wayID int64, nodeIDs []int64) error {
 
 	return nil
 }
+
+// QueryWay queries directed wayID by fromNodeID,toNodeID pair.
+// returned wayID: positive means travel forward following the fromNodeID,toNodeID sequence, negative means backward
+func (db *DB) QueryWay(fromNodeID, toNodeID int64) (int64, error) {
+	if db.db == nil {
+		return 0, errEmptyDB
+	}
+
+	var wayID int64
+	if err := db.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(defaultBucket))
+
+		v := b.Get(key(fromNodeID, toNodeID))
+		if v != nil {
+			wayID = parseValue(v)
+			return nil
+		}
+
+		// try again on backward
+		v = b.Get(key(toNodeID, fromNodeID))
+		if v == nil {
+			return errNotFound // both forward and backward not found
+		}
+		wayID = parseValue(v)
+		wayID = -wayID
+		return nil
+	}); err != nil {
+		return 0, err
+	}
+
+	return wayID, nil
+}
+
+// QueryWays queries directed wayIDs by nodeIDs.
+// `len(wayIDs) == len(nodeIDs)-1` since each way have to be decided by traveling from one node to another.
+// returned wayIDs: positive means travel forward following the nodeIDs sequence, negative means backward
+func (db *DB) QueryWays(nodeIDs []int64) ([]int64, error) {
+	if db.db == nil {
+		return nil, errEmptyDB
+	}
+
+	var wayIDs []int64
+	if err := db.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(defaultBucket))
+
+		for i := 0; i < len(nodeIDs)-1; i++ {
+			v := b.Get(key(nodeIDs[i], nodeIDs[i+1]))
+			if v != nil {
+				wayIDs = append(wayIDs, parseValue(v))
+				continue
+			}
+
+			// try again on backward
+			v = b.Get(key(nodeIDs[i+1], nodeIDs[i]))
+			if v == nil {
+				return errNotFound
+			}
+			wayID := parseValue(v)
+			wayIDs = append(wayIDs, -wayID)
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return wayIDs, nil
+}

From 3943cdac24381d3673a59af9c2e054b4ff974d13 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Thu, 9 Apr 2020 12:04:56 +0000
Subject: [PATCH 05/19] test: db ut

---
 .../util/waysnodes/nodes2wayblotdb/db_test.go | 156 ++++++++++++++++++
 1 file changed, 156 insertions(+)
 create mode 100644 integration/util/waysnodes/nodes2wayblotdb/db_test.go

diff --git a/integration/util/waysnodes/nodes2wayblotdb/db_test.go b/integration/util/waysnodes/nodes2wayblotdb/db_test.go
new file mode 100644
index 00000000000..cddafa91bdb
--- /dev/null
+++ b/integration/util/waysnodes/nodes2wayblotdb/db_test.go
@@ -0,0 +1,156 @@
+package nodes2wayblotdb
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"testing"
+)
+
+func testQuery(db *DB) error {
+	if db == nil {
+		return fmt.Errorf("invalid db")
+	}
+
+	queryWayCases := []struct {
+		fromNodeID int64
+		toNodeID   int64
+		wayID      int64
+	}{
+		{84760891102, 19496208102, 24418325},
+		{19496208102, 84760891102, -24418325},
+		{84762609102, 244183320001101, 24418332},
+		{244183320001101, 84762607102, 24418332},
+		{244183320001101, 84762609102, -24418332},
+		{84762607102, 244183320001101, -24418332},
+	}
+	for _, c := range queryWayCases {
+		wayID, err := db.QueryWay(c.fromNodeID, c.toNodeID)
+		if err != nil {
+			return err
+		}
+		if wayID != c.wayID {
+			return fmt.Errorf("query %d,%d in db, expect %d but got %d", c.fromNodeID, c.toNodeID, c.wayID, wayID)
+		}
+	}
+
+	queryWayExpectFailCases := []struct {
+		fromNodeID int64
+		toNodeID   int64
+	}{
+		{0, 0},
+		{84760891102, 84760891102},
+		{84762609102, 84762607102},
+	}
+	for _, c := range queryWayExpectFailCases {
+		wayID, err := db.QueryWay(c.fromNodeID, c.toNodeID)
+		if err == nil {
+			return fmt.Errorf("query %d,%d in db, expect fail but got %d", c.fromNodeID, c.toNodeID, wayID)
+		}
+	}
+
+	queryWaysCases := []struct {
+		nodeIDs []int64
+		wayIDs  []int64
+	}{
+		{
+			[]int64{84760891102, 19496208102}, []int64{24418325},
+		},
+		{
+			[]int64{19496208102, 84760891102}, []int64{-24418325},
+		},
+		{
+			[]int64{84762609102, 244183320001101, 84762607102}, []int64{24418332, 24418332},
+		},
+		{
+			[]int64{84762607102, 244183320001101, 84762609102}, []int64{-24418332, -24418332},
+		},
+		{
+			[]int64{84762609102, 244183320001101, 84762607199}, []int64{24418332, 24418330},
+		},
+		{
+			[]int64{84762607102, 244183320001101, 84762607199}, []int64{-24418332, 24418330},
+		},
+		{
+			[]int64{84762607199, 244183320001101, 84762607102}, []int64{-24418330, 24418332},
+		},
+		{
+			[]int64{84762607199, 244183320001101, 84762609102}, []int64{-24418330, -24418332},
+		},
+	}
+	for _, c := range queryWaysCases {
+		wayIDs, err := db.QueryWays(c.nodeIDs)
+		if err != nil {
+			return err
+		}
+		if !reflect.DeepEqual(wayIDs, c.wayIDs) {
+			return fmt.Errorf("query %v in db, expect %v but got %v", c.nodeIDs, c.wayIDs, wayIDs)
+		}
+	}
+
+	queryWaysExpectFailCases := []struct {
+		nodeIDs []int64
+	}{
+		{[]int64{0, 0}},
+		{[]int64{0, 0, 0}},
+		{[]int64{0, 1, 2}},
+		{[]int64{84760891102, 84762609102}},
+	}
+	for _, c := range queryWaysExpectFailCases {
+		wayIDs, err := db.QueryWays(c.nodeIDs)
+		if err == nil {
+			return fmt.Errorf("query %v in db, expect fail but got %v", c.nodeIDs, wayIDs)
+		}
+	}
+	return nil
+}
+
+func TestDB(t *testing.T) {
+
+	tempDBFile := "temp.db"
+
+	// Open a new DB.
+	db, err := Open(tempDBFile, false)
+	if err != nil {
+		t.Error(err)
+	}
+	defer os.Remove(db.db.Path()) // remove temp db file after test.
+
+	// Insert data.
+	if err := db.Write(24418325, []int64{84760891102, 19496208102}); err != nil {
+		t.Error(err)
+	}
+	if err := db.Write(24418332, []int64{84762609102, 244183320001101, 84762607102}); err != nil {
+		t.Error(err)
+	}
+	if err := db.Write(24418330, []int64{244183320001101, 84762607199}); err != nil {
+		t.Error(err)
+	}
+
+	// Query data.
+	if err := testQuery(db); err != nil {
+		t.Error(err)
+	}
+
+	// Close database to release the file lock.
+	if err := db.Close(); err != nil {
+		t.Error(err)
+	}
+	db = nil
+
+	// Open exist DB with readonly.
+	dbRead, err := Open(tempDBFile, true)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Query data.
+	if err := testQuery(dbRead); err != nil {
+		t.Error(err)
+	}
+
+	// Close database to release the file lock.
+	if err := dbRead.Close(); err != nil {
+		t.Error(err)
+	}
+}

From ff8c033ad4a591dcb44898549a809c722dd89a12 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Thu, 9 Apr 2020 12:05:16 +0000
Subject: [PATCH 06/19] chore: dependency for boltdb

---
 integration/go.mod | 1 +
 integration/go.sum | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/integration/go.mod b/integration/go.mod
index c005cfad0a4..7a28b115a75 100644
--- a/integration/go.mod
+++ b/integration/go.mod
@@ -10,5 +10,6 @@ require (
 	github.com/golang/snappy v0.0.1
 	github.com/qedus/osmpbf v1.1.0
 	github.com/twpayne/go-polyline v1.0.0
+	go.etcd.io/bbolt v1.3.4
 	google.golang.org/grpc v1.22.0
 )
diff --git a/integration/go.sum b/integration/go.sum
index efe2c9a7d57..9f11ab6dd0b 100644
--- a/integration/go.sum
+++ b/integration/go.sum
@@ -18,6 +18,8 @@ github.com/qedus/osmpbf v1.1.0 h1:1ewnhb7cX0VAp24M+ViDvLI9RKKgZOXFBLM5xGlB5TA=
 github.com/qedus/osmpbf v1.1.0/go.mod h1:37EgzlwZC2inPP5/rY1MZIxE6kgDof7MIljJuELs0c0=
 github.com/twpayne/go-polyline v1.0.0 h1:EA8HPg0MNS62R5D2E8B6zyz2TMkmkXVlQaVBVgY3F5A=
 github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
@@ -26,6 +28,7 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

From f27deba82fdce95ac28956ec7cddb20211a53667 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Thu, 9 Apr 2020 12:06:55 +0000
Subject: [PATCH 07/19] chore: fix dependency for boltdb

---
 integration/go.sum | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration/go.sum b/integration/go.sum
index 9f11ab6dd0b..b9a7f942f35 100644
--- a/integration/go.sum
+++ b/integration/go.sum
@@ -28,6 +28,7 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

From dd20e526395f7cbe897898abf013d389a8a16e7c Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Thu, 9 Apr 2020 12:53:47 +0000
Subject: [PATCH 08/19] feat: way2nodes csv utilities

---
 .../util/waysnodes/way2nodescsv/csv_line.go   |  66 +++++++++++
 .../waysnodes/way2nodescsv/csv_line_test.go   | 103 ++++++++++++++++++
 2 files changed, 169 insertions(+)
 create mode 100644 integration/util/waysnodes/way2nodescsv/csv_line.go
 create mode 100644 integration/util/waysnodes/way2nodescsv/csv_line_test.go

diff --git a/integration/util/waysnodes/way2nodescsv/csv_line.go b/integration/util/waysnodes/way2nodescsv/csv_line.go
new file mode 100644
index 00000000000..de89a0faeee
--- /dev/null
+++ b/integration/util/waysnodes/way2nodescsv/csv_line.go
@@ -0,0 +1,66 @@
+// Package way2nodescsv provides utilities to handle way2nodes mapping that stores by csv format/file.
+// Each record format is `wayID->nodeID,nodeID,nodeID,...`.
+package way2nodescsv
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// ParseRecord parses way2nodes csv record.
+func ParseRecord(record []string) (int64, []int64, error) {
+	if len(record) < 3 { // at least should be one wayID and two NodeIDs
+		return 0, nil, fmt.Errorf("invalid record %v", record)
+	}
+
+	wayID, err := strconv.ParseInt(record[0], 10, 64)
+	if err != nil {
+		return 0, nil, fmt.Errorf("decode wayID failed from %v", record)
+	}
+
+	nodeIDs := []int64{}
+	nodeElements := record[1:]
+	for _, nodeElement := range nodeElements {
+		if len(nodeElement) == 0 {
+			continue // the last element might be empty string
+		}
+
+		//nodeID
+		nodeID, err := strconv.ParseInt(nodeElement, 10, 64)
+		if err != nil {
+			return 0, nil, fmt.Errorf("decode nodeID failed from %s", nodeElement)
+		}
+		nodeIDs = append(nodeIDs, nodeID)
+	}
+	if len(nodeIDs) < 2 {
+		return 0, nil, fmt.Errorf("too less nodeIDs %v from %v", nodeIDs, nodeElements)
+	}
+
+	return wayID, nodeIDs, nil
+}
+
+// ParseLine parses way2nodes csv format line.
+func ParseLine(line string) (wayID int64, nodeIDs []int64, err error) {
+
+	record := strings.Split(line, ",")
+	return ParseRecord(record)
+}
+
+// FormatToRecord formats wayID and nodeIDs to csv record.
+func FormatToRecord(wayID int64, nodeIDs []int64) []string {
+
+	record := []string{}
+	record = append(record, strconv.FormatUint(uint64(wayID), 10))
+	for _, n := range nodeIDs {
+		record = append(record, strconv.FormatUint(uint64(n), 10))
+	}
+	return record
+}
+
+// FormatToString formats wayID and nodeIDs to csv string line.
+func FormatToString(wayID int64, nodeIDs []int64) string {
+
+	record := FormatToRecord(wayID, nodeIDs)
+	return strings.Join(record, ",")
+}
diff --git a/integration/util/waysnodes/way2nodescsv/csv_line_test.go b/integration/util/waysnodes/way2nodescsv/csv_line_test.go
new file mode 100644
index 00000000000..bf172d9336d
--- /dev/null
+++ b/integration/util/waysnodes/way2nodescsv/csv_line_test.go
@@ -0,0 +1,103 @@
+package way2nodescsv
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParseRecord(t *testing.T) {
+	cases := []struct {
+		record  []string
+		wayID   int64
+		nodeIDs []int64
+	}{
+		{strings.Split("24418325,84760891102,19496208102", ","), 24418325, []int64{84760891102, 19496208102}},
+		{strings.Split("24418325,84760891102,19496208102,", ","), 24418325, []int64{84760891102, 19496208102}},
+		{strings.Split("24418325,84760891102,19496208102,,,,,", ","), 24418325, []int64{84760891102, 19496208102}},
+	}
+
+	for _, c := range cases {
+		wayID, nodeIDs, err := ParseRecord(c.record)
+		if err != nil {
+			t.Error(err)
+		}
+		if wayID != c.wayID || !reflect.DeepEqual(nodeIDs, c.nodeIDs) {
+			t.Errorf("ParseRecord %v, expect %d,%v, but got %d,%v", c.record, c.wayID, c.nodeIDs, wayID, nodeIDs)
+		}
+	}
+
+	expectFailCases := []struct {
+		record []string
+	}{
+		{strings.Split("", ",")},
+		{strings.Split("24418325,84760891102", ",")},
+	}
+	for _, c := range expectFailCases {
+		wayID, nodeIDs, err := ParseRecord(c.record)
+		if err == nil {
+			t.Errorf("ParseRecord %v, expect fail but got %d,%v", c.record, wayID, nodeIDs)
+		}
+	}
+
+}
+
+func TestParseLine(t *testing.T) {
+
+	cases := []struct {
+		line    string
+		wayID   int64
+		nodeIDs []int64
+	}{
+		{"24418325,84760891102,19496208102", 24418325, []int64{84760891102, 19496208102}},
+		{"24418325,84760891102,19496208102,", 24418325, []int64{84760891102, 19496208102}},
+		{"24418325,84760891102,19496208102,,,,,", 24418325, []int64{84760891102, 19496208102}},
+	}
+
+	for _, c := range cases {
+		wayID, nodeIDs, err := ParseLine(c.line)
+		if err != nil {
+			t.Error(err)
+		}
+		if wayID != c.wayID || !reflect.DeepEqual(nodeIDs, c.nodeIDs) {
+			t.Errorf("ParseLine %s, expect %d,%v, but got %d,%v", c.line, c.wayID, c.nodeIDs, wayID, nodeIDs)
+		}
+	}
+
+	expectFailCases := []struct {
+		line string
+	}{
+		{""},
+		{"24418325,84760891102"},
+	}
+	for _, c := range expectFailCases {
+		wayID, nodeIDs, err := ParseLine(c.line)
+		if err == nil {
+			t.Errorf("ParseLine %s, expect fail but got %d,%v", c.line, wayID, nodeIDs)
+		}
+	}
+}
+
+func TestFormat(t *testing.T) {
+	cases := []struct {
+		wayID   int64
+		nodeIDs []int64
+		line    string
+	}{
+		{24418325, []int64{84760891102, 19496208102}, "24418325,84760891102,19496208102"},
+		{24418325, []int64{84760891102, 19496208102, 12345}, "24418325,84760891102,19496208102,12345"},
+	}
+
+	for _, c := range cases {
+		r := FormatToRecord(c.wayID, c.nodeIDs)
+		expectRecord := strings.Split(c.line, ",")
+		if !reflect.DeepEqual(r, expectRecord) {
+			t.Errorf("FormatToRecord %d,%v expect %v but got %v", c.wayID, c.nodeIDs, expectRecord, r)
+		}
+
+		line := FormatToString(c.wayID, c.nodeIDs)
+		if line != c.line {
+			t.Errorf("FormatToString %d,%v expect %s but got %s", c.wayID, c.nodeIDs, c.line, line)
+		}
+	}
+}

From 4aad342461fe48b3b269f8708b6bfc82aae33bd5 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 01:29:17 +0000
Subject: [PATCH 09/19] feat: add new nodes2way-db-builder to extract way-nodes
 from csv and insert into db

---
 integration/cmd/nodes2way-db-builder/flags.go | 17 ++++++
 integration/cmd/nodes2way-db-builder/main.go  | 42 ++++++++++++++
 integration/cmd/nodes2way-db-builder/read.go  | 58 +++++++++++++++++++
 integration/cmd/nodes2way-db-builder/store.go | 41 +++++++++++++
 integration/util/waysnodes/way_nodes.go       |  7 +++
 5 files changed, 165 insertions(+)
 create mode 100644 integration/cmd/nodes2way-db-builder/flags.go
 create mode 100644 integration/cmd/nodes2way-db-builder/main.go
 create mode 100644 integration/cmd/nodes2way-db-builder/read.go
 create mode 100644 integration/cmd/nodes2way-db-builder/store.go
 create mode 100644 integration/util/waysnodes/way_nodes.go

diff --git a/integration/cmd/nodes2way-db-builder/flags.go b/integration/cmd/nodes2way-db-builder/flags.go
new file mode 100644
index 00000000000..b4233ba52cc
--- /dev/null
+++ b/integration/cmd/nodes2way-db-builder/flags.go
@@ -0,0 +1,17 @@
+package main
+
+import (
+	"flag"
+)
+
+var flags struct {
+	snappyCompressed bool
+	in               string // E.g., wayid2nodeids.csv or wayid2nodeids.csv.snappy
+	out              string // DB path
+}
+
+func init() {
+	flag.StringVar(&flags.in, "i", "wayid2nodeids.csv.snappy", "Input wayid2nodeids csv file path, e.g., wayid2nodeids.csv or wayid2nodeids.csv.snappy.")
+	flag.StringVar(&flags.out, "o", "nodes2way.db", "Output DB file path.")
+	flag.BoolVar(&flags.snappyCompressed, "snappy-compressed", true, "Whether input csv snappy compressed or not.")
+}
diff --git a/integration/cmd/nodes2way-db-builder/main.go b/integration/cmd/nodes2way-db-builder/main.go
new file mode 100644
index 00000000000..140a23e2c4f
--- /dev/null
+++ b/integration/cmd/nodes2way-db-builder/main.go
@@ -0,0 +1,42 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"time"
+
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes"
+	"github.com/golang/glog"
+)
+
+func main() {
+	flag.Parse()
+	defer glog.Flush()
+
+	pipeline()
+}
+
+func pipeline() {
+	startTime := time.Now()
+
+	wayNodesChan := make(chan *waysnodes.WayNodes)
+	errChan := make(chan error)
+
+	go func() {
+		errChan <- newStore(wayNodesChan, flags.out)
+	}()
+	go func() {
+		errChan <- newReader(flags.in, flags.snappyCompressed, wayNodesChan)
+		close(wayNodesChan)
+	}()
+
+	for i := 0; i < 2; i++ {
+		if err := <-errChan; err != nil {
+			glog.Error(err)
+			os.Exit(1)
+			return
+		}
+	}
+
+	glog.Infof("%s totally takes %f seconds for processing.", os.Args[0], time.Now().Sub(startTime).Seconds())
+}
diff --git a/integration/cmd/nodes2way-db-builder/read.go b/integration/cmd/nodes2way-db-builder/read.go
new file mode 100644
index 00000000000..71896632ebd
--- /dev/null
+++ b/integration/cmd/nodes2way-db-builder/read.go
@@ -0,0 +1,58 @@
+package main
+
+import (
+	"encoding/csv"
+	"io"
+	"os"
+	"time"
+
+	"github.com/golang/snappy"
+
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes"
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes/way2nodescsv"
+	"github.com/golang/glog"
+)
+
+func newReader(in string, snappyCompressed bool, out chan<- *waysnodes.WayNodes) error {
+	startTime := time.Now()
+
+	f, err := os.Open(in)
+	defer f.Close()
+	if err != nil {
+		return err
+	}
+	glog.V(1).Infof("open %s succeed.\n", in)
+
+	var r *csv.Reader
+	if snappyCompressed {
+		r = csv.NewReader(snappy.NewReader(f))
+	} else {
+		r = csv.NewReader(f)
+	}
+	r.ReuseRecord = true
+	r.FieldsPerRecord = -1 // disable fields count check
+
+	var total, succeed int
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		total++
+
+		wayNodes := waysnodes.WayNodes{}
+		wayNodes.WayID, wayNodes.NodeIDs, err = way2nodescsv.ParseRecord(record)
+		if err != nil {
+			glog.Warningf("Parse record %v failed, err: %v", record, err)
+			continue
+		}
+		out <- &wayNodes
+		succeed++
+	}
+
+	glog.V(1).Infof("Read wayID,nodeID,nodeID,... from file %s, total count %d, succeed parsed %d, takes %f seconds", in, total, succeed, time.Now().Sub(startTime).Seconds())
+	return nil
+}
diff --git a/integration/cmd/nodes2way-db-builder/store.go b/integration/cmd/nodes2way-db-builder/store.go
new file mode 100644
index 00000000000..df3b5e90675
--- /dev/null
+++ b/integration/cmd/nodes2way-db-builder/store.go
@@ -0,0 +1,41 @@
+package main
+
+import (
+	"time"
+
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes"
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes/nodes2wayblotdb"
+	"github.com/golang/glog"
+)
+
+func newStore(in <-chan *waysnodes.WayNodes, out string) (err error) {
+	startTime := time.Now()
+
+	db, err := nodes2wayblotdb.Open(out, false)
+	if err != nil {
+		return
+	}
+	defer func() {
+		err = db.Close()
+	}()
+
+	var inCount, succeedCount int
+	for {
+		wayNodes, ok := <-in
+		if !ok {
+			break
+		}
+		inCount++
+
+		if err := db.Write(wayNodes.WayID, wayNodes.NodeIDs); err != nil {
+			if glog.V(3) { // avoid affect performance by verbose log
+				glog.Infof("Update %+v into db failed, err: %v", wayNodes, err)
+			}
+			continue
+		}
+		succeedCount++
+	}
+
+	glog.V(1).Infof("Built DB %s, in count %d, succeed count %d, takes %f seconds", out, inCount, succeedCount, time.Now().Sub(startTime).Seconds())
+	return
+}
diff --git a/integration/util/waysnodes/way_nodes.go b/integration/util/waysnodes/way_nodes.go
new file mode 100644
index 00000000000..96479c6bfe8
--- /dev/null
+++ b/integration/util/waysnodes/way_nodes.go
@@ -0,0 +1,7 @@
+package waysnodes
+
+// WayNodes packages wayID and nodeIDs together.
+type WayNodes struct {
+	WayID   int64
+	NodeIDs []int64
+}

From 34c9939828178923ac02a377870679a3160bd8a3 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 01:30:45 +0000
Subject: [PATCH 10/19] feat: manually sync db to improve write performance

---
 integration/util/waysnodes/nodes2wayblotdb/db.go | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/integration/util/waysnodes/nodes2wayblotdb/db.go b/integration/util/waysnodes/nodes2wayblotdb/db.go
index 25eeb8c8fe3..d12ece90d02 100644
--- a/integration/util/waysnodes/nodes2wayblotdb/db.go
+++ b/integration/util/waysnodes/nodes2wayblotdb/db.go
@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/golang/glog"
+
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -47,6 +49,10 @@ func Open(dbFilePath string, readOnly bool) (*DB, error) {
 		return &db, nil
 	}
 
+	// to improve write performance, but will manually sync before close
+	db.db.NoSync = true
+	db.db.NoFreelistSync = true
+
 	// for write, make sure bucket available
 	err = db.db.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(defaultBucket))
@@ -68,6 +74,13 @@ func (db *DB) Close() error {
 		return errEmptyDB
 	}
 
+	if !db.db.IsReadOnly() {
+		if err := db.db.Sync(); err != nil {
+			glog.Error(err)
+			//return err	// don't return since we still hope the Close can be called
+		}
+	}
+
 	return db.db.Close()
 }
 

From c21d52957ae6de98ac93e2f0fc0b96ca1e4dc2ef Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 01:32:21 +0000
Subject: [PATCH 11/19] chore: ignore new binary and vscode config for source
 code management

---
 integration/.gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/integration/.gitignore b/integration/.gitignore
index 7c4515c9852..7f1e2b143ab 100644
--- a/integration/.gitignore
+++ b/integration/.gitignore
@@ -1,4 +1,7 @@
 
+# vscode
+.vscode/
+
 # binaries
 cmd/__debug_bin
 cmd/*/__debug_bin
@@ -10,6 +13,7 @@ cmd/osrm-ranking/osrm-ranking
 cmd/trafficcache-parallel-test/trafficcache-parallel-test
 cmd/osrm-files-extractor/osrm-files-extractor
 cmd/historicalspeed-timezone-builder/historicalspeed-timezone-builder
+cmd/nodes2way-db-builder/nodes2way-db-builder
 
 # test files
 cmd/trafficproxy-cli/*_flows.csv

From cedcc300fc26fba306cf35884408baae9d121abe Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 02:29:21 +0000
Subject: [PATCH 12/19] fix: improve error info

---
 integration/util/waysnodes/nodes2wayblotdb/db.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/integration/util/waysnodes/nodes2wayblotdb/db.go b/integration/util/waysnodes/nodes2wayblotdb/db.go
index d12ece90d02..dabd7258e89 100644
--- a/integration/util/waysnodes/nodes2wayblotdb/db.go
+++ b/integration/util/waysnodes/nodes2wayblotdb/db.go
@@ -20,8 +20,8 @@ const (
 )
 
 var (
-	errEmptyDB  = errors.New("empty db")
-	errNotFound = errors.New("not found")
+	errEmptyDB     = errors.New("empty db")
+	errKeyNotFound = errors.New("key not found")
 )
 
 // Open creates/opens a DB structure to store the nodes2way mapping.
@@ -134,7 +134,7 @@ func (db *DB) QueryWay(fromNodeID, toNodeID int64) (int64, error) {
 		// try again on backward
 		v = b.Get(key(toNodeID, fromNodeID))
 		if v == nil {
-			return errNotFound // both forward and backward not found
+			return errKeyNotFound // both forward and backward not found
 		}
 		wayID = parseValue(v)
 		wayID = -wayID
@@ -168,7 +168,7 @@ func (db *DB) QueryWays(nodeIDs []int64) ([]int64, error) {
 			// try again on backward
 			v = b.Get(key(nodeIDs[i+1], nodeIDs[i]))
 			if v == nil {
-				return errNotFound
+				return errKeyNotFound
 			}
 			wayID := parseValue(v)
 			wayIDs = append(wayIDs, -wayID)

From f10aaf2a6ace6d4ca08d6ff47f465b27ce085d6d Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 02:37:19 +0000
Subject: [PATCH 13/19] feat: cli to query ways from nodes

---
 integration/.gitignore                 |  1 +
 integration/cmd/nodes2way-cli/flags.go | 17 +++++++++++
 integration/cmd/nodes2way-cli/main.go  | 41 ++++++++++++++++++++++++++
 3 files changed, 59 insertions(+)
 create mode 100644 integration/cmd/nodes2way-cli/flags.go
 create mode 100644 integration/cmd/nodes2way-cli/main.go

diff --git a/integration/.gitignore b/integration/.gitignore
index 7f1e2b143ab..64df1abcb38 100644
--- a/integration/.gitignore
+++ b/integration/.gitignore
@@ -14,6 +14,7 @@ cmd/trafficcache-parallel-test/trafficcache-parallel-test
 cmd/osrm-files-extractor/osrm-files-extractor
 cmd/historicalspeed-timezone-builder/historicalspeed-timezone-builder
 cmd/nodes2way-db-builder/nodes2way-db-builder
+cmd/nodes2way-cli/nodes2way-cli
 
 # test files
 cmd/trafficproxy-cli/*_flows.csv
diff --git a/integration/cmd/nodes2way-cli/flags.go b/integration/cmd/nodes2way-cli/flags.go
new file mode 100644
index 00000000000..f66c68a57e0
--- /dev/null
+++ b/integration/cmd/nodes2way-cli/flags.go
@@ -0,0 +1,17 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/Telenav/osrm-backend/integration/pkg/wayidsflag"
+)
+
+var flags struct {
+	nodeIDs wayidsflag.WayIDs //TODO: will refactor it to intsflag.Int64s later
+	db      string
+}
+
+func init() {
+	flag.StringVar(&flags.db, "db", "nodes2way.db", "DB file path.")
+	flag.Var(&flags.nodeIDs, "nodes", "Continuously comma-seperated nodeIDs on a route. E.g. '829733412,104489539'.")
+}
diff --git a/integration/cmd/nodes2way-cli/main.go b/integration/cmd/nodes2way-cli/main.go
new file mode 100644
index 00000000000..240014db1be
--- /dev/null
+++ b/integration/cmd/nodes2way-cli/main.go
@@ -0,0 +1,41 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes/nodes2wayblotdb"
+)
+
+// output logs to stderr without timestamp
+var cliLog = log.New(os.Stderr, "", 0)
+
+func main() {
+	flag.Parse()
+
+	wayIDs, err := query(flags.db, flags.nodeIDs)
+	if err != nil {
+		cliLog.Println(err)
+		os.Exit(1)
+		return
+	}
+	fmt.Println(wayIDs)
+}
+
+func query(dbFile string, nodeIDs []int64) ([]int64, error) {
+
+	db, err := nodes2wayblotdb.Open(dbFile, true)
+	if err != nil {
+		return nil, err
+	}
+	defer db.Close()
+
+	wayIDs, err := db.QueryWays(nodeIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	return wayIDs, nil
+}

From e01781d46d4310797d8967c3cc4cf0112b3e74f9 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 02:38:19 +0000
Subject: [PATCH 14/19] refactor: rename cmd

---
 integration/.gitignore                                          | 2 +-
 .../cmd/{nodes2way-db-builder => nodes2way-builder}/flags.go    | 0
 .../cmd/{nodes2way-db-builder => nodes2way-builder}/main.go     | 0
 .../cmd/{nodes2way-db-builder => nodes2way-builder}/read.go     | 0
 .../cmd/{nodes2way-db-builder => nodes2way-builder}/store.go    | 0
 5 files changed, 1 insertion(+), 1 deletion(-)
 rename integration/cmd/{nodes2way-db-builder => nodes2way-builder}/flags.go (100%)
 rename integration/cmd/{nodes2way-db-builder => nodes2way-builder}/main.go (100%)
 rename integration/cmd/{nodes2way-db-builder => nodes2way-builder}/read.go (100%)
 rename integration/cmd/{nodes2way-db-builder => nodes2way-builder}/store.go (100%)

diff --git a/integration/.gitignore b/integration/.gitignore
index 64df1abcb38..2908cde9314 100644
--- a/integration/.gitignore
+++ b/integration/.gitignore
@@ -13,7 +13,7 @@ cmd/osrm-ranking/osrm-ranking
 cmd/trafficcache-parallel-test/trafficcache-parallel-test
 cmd/osrm-files-extractor/osrm-files-extractor
 cmd/historicalspeed-timezone-builder/historicalspeed-timezone-builder
-cmd/nodes2way-db-builder/nodes2way-db-builder
+cmd/nodes2way-builder/nodes2way-builder
 cmd/nodes2way-cli/nodes2way-cli
 
 # test files
diff --git a/integration/cmd/nodes2way-db-builder/flags.go b/integration/cmd/nodes2way-builder/flags.go
similarity index 100%
rename from integration/cmd/nodes2way-db-builder/flags.go
rename to integration/cmd/nodes2way-builder/flags.go
diff --git a/integration/cmd/nodes2way-db-builder/main.go b/integration/cmd/nodes2way-builder/main.go
similarity index 100%
rename from integration/cmd/nodes2way-db-builder/main.go
rename to integration/cmd/nodes2way-builder/main.go
diff --git a/integration/cmd/nodes2way-db-builder/read.go b/integration/cmd/nodes2way-builder/read.go
similarity index 100%
rename from integration/cmd/nodes2way-db-builder/read.go
rename to integration/cmd/nodes2way-builder/read.go
diff --git a/integration/cmd/nodes2way-db-builder/store.go b/integration/cmd/nodes2way-builder/store.go
similarity index 100%
rename from integration/cmd/nodes2way-db-builder/store.go
rename to integration/cmd/nodes2way-builder/store.go

From e91fe382ef7d43c18a6079ab979a138894c82cd1 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 02:43:14 +0000
Subject: [PATCH 15/19] feat: measure querying time cost, only output by glog
 which will not display in stdout/stderr by default

---
 integration/cmd/nodes2way-cli/main.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/integration/cmd/nodes2way-cli/main.go b/integration/cmd/nodes2way-cli/main.go
index 240014db1be..764222acc78 100644
--- a/integration/cmd/nodes2way-cli/main.go
+++ b/integration/cmd/nodes2way-cli/main.go
@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"time"
 
 	"github.com/Telenav/osrm-backend/integration/util/waysnodes/nodes2wayblotdb"
+	"github.com/golang/glog"
 )
 
 // output logs to stderr without timestamp
@@ -32,10 +34,14 @@ func query(dbFile string, nodeIDs []int64) ([]int64, error) {
 	}
 	defer db.Close()
 
+	startTime := time.Now()
+
 	wayIDs, err := db.QueryWays(nodeIDs)
 	if err != nil {
 		return nil, err
 	}
 
+	glog.Infof("Querying takes %f seconds", time.Now().Sub(startTime).Seconds()) // easy to measure querying time cost
+
 	return wayIDs, nil
 }

From 04da7afccbdeb9846a05d427c4d05bf4ecc08238 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 03:14:38 +0000
Subject: [PATCH 16/19] feat: print db statitics

---
 integration/cmd/nodes2way-cli/flags.go        |  4 ++-
 integration/cmd/nodes2way-cli/main.go         | 21 ++++++++++++
 .../util/waysnodes/nodes2wayblotdb/db.go      | 32 +++++++++++++++++++
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/integration/cmd/nodes2way-cli/flags.go b/integration/cmd/nodes2way-cli/flags.go
index f66c68a57e0..11f0ef7b7b8 100644
--- a/integration/cmd/nodes2way-cli/flags.go
+++ b/integration/cmd/nodes2way-cli/flags.go
@@ -9,9 +9,11 @@ import (
 var flags struct {
 	nodeIDs wayidsflag.WayIDs //TODO: will refactor it to intsflag.Int64s later
 	db      string
+	dbStat  bool
 }
 
 func init() {
+	flag.Var(&flags.nodeIDs, "nodes", "Continuously comma-seperated nodeIDs on a route. E.g. '167772220006101,167772220007101,167772220008101'.")
 	flag.StringVar(&flags.db, "db", "nodes2way.db", "DB file path.")
-	flag.Var(&flags.nodeIDs, "nodes", "Continuously comma-seperated nodeIDs on a route. E.g. '829733412,104489539'.")
+	flag.BoolVar(&flags.dbStat, "dbstat", false, "Print DB statistics.")
 }
diff --git a/integration/cmd/nodes2way-cli/main.go b/integration/cmd/nodes2way-cli/main.go
index 764222acc78..74c1d22847b 100644
--- a/integration/cmd/nodes2way-cli/main.go
+++ b/integration/cmd/nodes2way-cli/main.go
@@ -17,6 +17,17 @@ var cliLog = log.New(os.Stderr, "", 0)
 func main() {
 	flag.Parse()
 
+	if flags.dbStat {
+		s, err := dbStat(flags.db)
+		if err != nil {
+			cliLog.Println(err)
+			os.Exit(1)
+			return
+		}
+		fmt.Println(s)
+		return
+	}
+
 	wayIDs, err := query(flags.db, flags.nodeIDs)
 	if err != nil {
 		cliLog.Println(err)
@@ -45,3 +56,13 @@ func query(dbFile string, nodeIDs []int64) ([]int64, error) {
 
 	return wayIDs, nil
 }
+
+func dbStat(dbFile string) (string, error) {
+	db, err := nodes2wayblotdb.Open(dbFile, true)
+	if err != nil {
+		return "", err
+	}
+	defer db.Close()
+
+	return db.Statistics(), nil
+}
diff --git a/integration/util/waysnodes/nodes2wayblotdb/db.go b/integration/util/waysnodes/nodes2wayblotdb/db.go
index dabd7258e89..353a8afb4d9 100644
--- a/integration/util/waysnodes/nodes2wayblotdb/db.go
+++ b/integration/util/waysnodes/nodes2wayblotdb/db.go
@@ -2,6 +2,7 @@
 package nodes2wayblotdb
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 
@@ -84,6 +85,37 @@ func (db *DB) Close() error {
 	return db.db.Close()
 }
 
+// Statistics returns internal statistics information.
+func (db *DB) Statistics() string {
+	if db.db == nil {
+		return ""
+	}
+
+	s := struct {
+		DBStat     string `json:"dbstat"`
+		BucketStat string `json:"bucketstat"`
+	}{}
+
+	s.DBStat = fmt.Sprintf("%+v", db.db.Stats())
+	if err := db.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(defaultBucket))
+
+		s.BucketStat = fmt.Sprintf("%+v", b.Stats()) // bucket stats
+		return nil
+	}); err != nil {
+		glog.Error(err)
+		return err.Error()
+	}
+
+	b, err := json.Marshal(&s)
+	if err != nil {
+		glog.Error(err)
+		return err.Error()
+	}
+
+	return string(b)
+}
+
 // Write writes wayID and its nodeIDs into cache or storage.
 // wayID: is undirected when input, so will always be positive.
 func (db *DB) Write(wayID int64, nodeIDs []int64) error {

From 6e87f777bf88a51e62813eb1d9fea11c0ea9435f Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 05:50:12 +0000
Subject: [PATCH 17/19] feat: more specified error message

---
 integration/util/waysnodes/nodes2wayblotdb/db.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/util/waysnodes/nodes2wayblotdb/db.go b/integration/util/waysnodes/nodes2wayblotdb/db.go
index 353a8afb4d9..c852babbb9d 100644
--- a/integration/util/waysnodes/nodes2wayblotdb/db.go
+++ b/integration/util/waysnodes/nodes2wayblotdb/db.go
@@ -200,7 +200,7 @@ func (db *DB) QueryWays(nodeIDs []int64) ([]int64, error) {
 			// try again on backward
 			v = b.Get(key(nodeIDs[i+1], nodeIDs[i]))
 			if v == nil {
-				return errKeyNotFound
+				return fmt.Errorf("%v: %d,%d", errKeyNotFound, nodeIDs[i], nodeIDs[i+1])
 			}
 			wayID := parseValue(v)
 			wayIDs = append(wayIDs, -wayID)

From cc2d5bc8d8b75bf0726d1dedb7ccefce1d389f0a Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Fri, 10 Apr 2020 08:33:31 +0000
Subject: [PATCH 18/19] docs: add two cli tools

---
 integration/README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/integration/README.md b/integration/README.md
index 6bddf5c40ec..d387f3ea1a4 100644
--- a/integration/README.md
+++ b/integration/README.md
@@ -11,6 +11,12 @@ Command line tool [cmd/osrm-traffic-updater](cmd/osrm-traffic-updater/) is desig
 ## wayid2nodeids_extractor
 Command line tool for extract wayid to nodeids mapping from PBF. Code in [cmd/wayid2nodeid-extractor](cmd/wayid2nodeid-extractor/).        
 
+## nodes2way-builder
+Command line tool for build nodes2way mapping DB. Code in [cmd/nodes2way-builder](cmd/nodes2way-builder/).        
+
+## nodes2way-cli
+Command line tool for querying wayids from nodeids in DB. Code in [cmd/nodes2way-cli](cmd/nodes2way-cli/).        
+
 ## snappy
 Command line tool for [snappy](github.com/golang/snappy) compression. Code in [cmd/snappy](cmd/snappy/).  
 

From d53ba823b34564cff7888912e79e58ab80eed520 Mon Sep 17 00:00:00 2001
From: Jay Zhang <wangyoucao577@gmail.com>
Date: Mon, 13 Apr 2020 02:02:35 +0000
Subject: [PATCH 19/19] refactor: rename bucket name

---
 integration/util/waysnodes/nodes2wayblotdb/db.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/integration/util/waysnodes/nodes2wayblotdb/db.go b/integration/util/waysnodes/nodes2wayblotdb/db.go
index c852babbb9d..dad7bbeea0b 100644
--- a/integration/util/waysnodes/nodes2wayblotdb/db.go
+++ b/integration/util/waysnodes/nodes2wayblotdb/db.go
@@ -17,7 +17,7 @@ type DB struct {
 }
 
 const (
-	defaultBucket = "bucket"
+	nodes2WayBucket = "nodes2way_bucket"
 )
 
 var (
@@ -56,7 +56,7 @@ func Open(dbFilePath string, readOnly bool) (*DB, error) {
 
 	// for write, make sure bucket available
 	err = db.db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(defaultBucket))
+		_, err := tx.CreateBucketIfNotExists([]byte(nodes2WayBucket))
 		if err != nil {
 			return fmt.Errorf("failed to create bucket: %s", err)
 		}
@@ -98,7 +98,7 @@ func (db *DB) Statistics() string {
 
 	s.DBStat = fmt.Sprintf("%+v", db.db.Stats())
 	if err := db.db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(defaultBucket))
+		b := tx.Bucket([]byte(nodes2WayBucket))
 
 		s.BucketStat = fmt.Sprintf("%+v", b.Stats()) // bucket stats
 		return nil
@@ -130,7 +130,7 @@ func (db *DB) Write(wayID int64, nodeIDs []int64) error {
 	}
 
 	err := db.db.Update(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(defaultBucket))
+		b := tx.Bucket([]byte(nodes2WayBucket))
 
 		for i := 0; i < len(nodeIDs)-1; i++ {
 			if err := b.Put(key(nodeIDs[i], nodeIDs[i+1]), value(wayID)); err != nil {
@@ -155,7 +155,7 @@ func (db *DB) QueryWay(fromNodeID, toNodeID int64) (int64, error) {
 
 	var wayID int64
 	if err := db.db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(defaultBucket))
+		b := tx.Bucket([]byte(nodes2WayBucket))
 
 		v := b.Get(key(fromNodeID, toNodeID))
 		if v != nil {
@@ -188,7 +188,7 @@ func (db *DB) QueryWays(nodeIDs []int64) ([]int64, error) {
 
 	var wayIDs []int64
 	if err := db.db.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(defaultBucket))
+		b := tx.Bucket([]byte(nodes2WayBucket))
 
 		for i := 0; i < len(nodeIDs)-1; i++ {
 			v := b.Get(key(nodeIDs[i], nodeIDs[i+1]))