Generate nodes2ways DB

integration/.gitignore

@@ -1,4 +1,7 @@
 
+# vscode
+.vscode/
+
 # binaries
 cmd/__debug_bin
 cmd/*/__debug_bin
@@ -10,6 +13,8 @@ cmd/osrm-ranking/osrm-ranking
 cmd/trafficcache-parallel-test/trafficcache-parallel-test
 cmd/osrm-files-extractor/osrm-files-extractor
 cmd/historicalspeed-timezone-builder/historicalspeed-timezone-builder
+cmd/nodes2way-builder/nodes2way-builder
+cmd/nodes2way-cli/nodes2way-cli
 
 # test files
 cmd/trafficproxy-cli/*_flows.csv

integration/README.md

@@ -11,6 +11,12 @@ Command line tool [cmd/osrm-traffic-updater](cmd/osrm-traffic-updater/) is desig
 ## wayid2nodeids_extractor
 Command line tool for extract wayid to nodeids mapping from PBF. Code in [cmd/wayid2nodeid-extractor](cmd/wayid2nodeid-extractor/).        
 
+## nodes2way-builder
+Command line tool for build nodes2way mapping DB. Code in [cmd/nodes2way-builder](cmd/nodes2way-builder/).        
+
+## nodes2way-cli
+Command line tool for querying wayids from nodeids in DB. Code in [cmd/nodes2way-cli](cmd/nodes2way-cli/).        
+
 ## snappy
 Command line tool for [snappy](github.com/golang/snappy) compression. Code in [cmd/snappy](cmd/snappy/).  
 

integration/cmd/nodes2way-builder/flags.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"flag"
+)
+
+var flags struct {
+	snappyCompressed bool
+	in               string // E.g., wayid2nodeids.csv or wayid2nodeids.csv.snappy
+	out              string // DB path
+}
+
+func init() {
+	flag.StringVar(&flags.in, "i", "wayid2nodeids.csv.snappy", "Input wayid2nodeids csv file path, e.g., wayid2nodeids.csv or wayid2nodeids.csv.snappy.")
+	flag.StringVar(&flags.out, "o", "nodes2way.db", "Output DB file path.")
+	flag.BoolVar(&flags.snappyCompressed, "snappy-compressed", true, "Whether input csv snappy compressed or not.")
+}

integration/cmd/nodes2way-builder/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"time"
+
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes"
+	"github.com/golang/glog"
+)
+
+func main() {
+	flag.Parse()
+	defer glog.Flush()
+
+	pipeline()
+}
+
+func pipeline() {
+	startTime := time.Now()
+
+	wayNodesChan := make(chan *waysnodes.WayNodes)
+	errChan := make(chan error)
+
+	go func() {
+		errChan <- newStore(wayNodesChan, flags.out)
+	}()
+	go func() {
+		errChan <- newReader(flags.in, flags.snappyCompressed, wayNodesChan)
+		close(wayNodesChan)
+	}()
+
+	for i := 0; i < 2; i++ {
+		if err := <-errChan; err != nil {
+			glog.Error(err)
+			os.Exit(1)
+			return
+		}
+	}
+
+	glog.Infof("%s totally takes %f seconds for processing.", os.Args[0], time.Now().Sub(startTime).Seconds())
+}

integration/cmd/nodes2way-builder/read.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"encoding/csv"
+	"io"
+	"os"
+	"time"
+
+	"github.com/golang/snappy"
+
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes"
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes/way2nodescsv"
+	"github.com/golang/glog"
+)
+
+func newReader(in string, snappyCompressed bool, out chan<- *waysnodes.WayNodes) error {
+	startTime := time.Now()
+
+	f, err := os.Open(in)
+	defer f.Close()
+	if err != nil {
+		return err
+	}
+	glog.V(1).Infof("open %s succeed.\n", in)
+
+	var r *csv.Reader
+	if snappyCompressed {
+		r = csv.NewReader(snappy.NewReader(f))
+	} else {
+		r = csv.NewReader(f)
+	}
+	r.ReuseRecord = true
+	r.FieldsPerRecord = -1 // disable fields count check
+
+	var total, succeed int
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		total++
+
+		wayNodes := waysnodes.WayNodes{}
+		wayNodes.WayID, wayNodes.NodeIDs, err = way2nodescsv.ParseRecord(record)
+		if err != nil {
+			glog.Warningf("Parse record %v failed, err: %v", record, err)
+			continue
+		}
+		out <- &wayNodes
+		succeed++
+	}
+
+	glog.V(1).Infof("Read wayID,nodeID,nodeID,... from file %s, total count %d, succeed parsed %d, takes %f seconds", in, total, succeed, time.Now().Sub(startTime).Seconds())
+	return nil
+}

integration/cmd/nodes2way-builder/store.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"time"
+
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes"
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes/nodes2wayblotdb"
+	"github.com/golang/glog"
+)
+
+func newStore(in <-chan *waysnodes.WayNodes, out string) (err error) {
+	startTime := time.Now()
+
+	db, err := nodes2wayblotdb.Open(out, false)
+	if err != nil {
+		return
+	}
+	defer func() {
+		err = db.Close()
+	}()
+
+	var inCount, succeedCount int
+	for {
+		wayNodes, ok := <-in
+		if !ok {
+			break
+		}
+		inCount++
+
+		if err := db.Write(wayNodes.WayID, wayNodes.NodeIDs); err != nil {
+			if glog.V(3) { // avoid affect performance by verbose log
+				glog.Infof("Update %+v into db failed, err: %v", wayNodes, err)
+			}
+			continue
+		}
+		succeedCount++
+	}
+
+	glog.V(1).Infof("Built DB %s, in count %d, succeed count %d, takes %f seconds", out, inCount, succeedCount, time.Now().Sub(startTime).Seconds())
+	return
+}

integration/cmd/nodes2way-cli/flags.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/Telenav/osrm-backend/integration/pkg/wayidsflag"
+)
+
+var flags struct {
+	nodeIDs wayidsflag.WayIDs //TODO: will refactor it to intsflag.Int64s later
+	db      string
+	dbStat  bool
+}
+
+func init() {
+	flag.Var(&flags.nodeIDs, "nodes", "Continuously comma-seperated nodeIDs on a route. E.g. '167772220006101,167772220007101,167772220008101'.")
+	flag.StringVar(&flags.db, "db", "nodes2way.db", "DB file path.")
+	flag.BoolVar(&flags.dbStat, "dbstat", false, "Print DB statistics.")
+}

integration/cmd/nodes2way-cli/main.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/Telenav/osrm-backend/integration/util/waysnodes/nodes2wayblotdb"
+	"github.com/golang/glog"
+)
+
+// output logs to stderr without timestamp
+var cliLog = log.New(os.Stderr, "", 0)
+
+func main() {
+	flag.Parse()
+
+	if flags.dbStat {
+		s, err := dbStat(flags.db)
+		if err != nil {
+			cliLog.Println(err)
+			os.Exit(1)
+			return
+		}
+		fmt.Println(s)
+		return
+	}
+
+	wayIDs, err := query(flags.db, flags.nodeIDs)
+	if err != nil {
+		cliLog.Println(err)
+		os.Exit(1)
+		return
+	}
+	fmt.Println(wayIDs)
+}
+
+func query(dbFile string, nodeIDs []int64) ([]int64, error) {
+
+	db, err := nodes2wayblotdb.Open(dbFile, true)
+	if err != nil {
+		return nil, err
+	}
+	defer db.Close()
+
+	startTime := time.Now()
+
+	wayIDs, err := db.QueryWays(nodeIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	glog.Infof("Querying takes %f seconds", time.Now().Sub(startTime).Seconds()) // easy to measure querying time cost
+
+	return wayIDs, nil
+}
+
+func dbStat(dbFile string) (string, error) {
+	db, err := nodes2wayblotdb.Open(dbFile, true)
+	if err != nil {
+		return "", err
+	}
+	defer db.Close()
+
+	return db.Statistics(), nil
+}

integration/go.mod

@@ -10,5 +10,6 @@ require (
 	github.com/golang/snappy v0.0.1
 	github.com/qedus/osmpbf v1.1.0
 	github.com/twpayne/go-polyline v1.0.0
+	go.etcd.io/bbolt v1.3.4
 	google.golang.org/grpc v1.22.0
 )

integration/go.sum

@@ -18,6 +18,8 @@ github.com/qedus/osmpbf v1.1.0 h1:1ewnhb7cX0VAp24M+ViDvLI9RKKgZOXFBLM5xGlB5TA=
 github.com/qedus/osmpbf v1.1.0/go.mod h1:37EgzlwZC2inPP5/rY1MZIxE6kgDof7MIljJuELs0c0=
 github.com/twpayne/go-polyline v1.0.0 h1:EA8HPg0MNS62R5D2E8B6zyz2TMkmkXVlQaVBVgY3F5A=
 github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
@@ -26,6 +28,8 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=