observe .osrm.nbg_nodes

integration/cmd/osrm-files-extractor/flags.go

@@ -6,10 +6,12 @@ var flags struct {
 	filePath     string
 	singleFile   bool
 	printSummary int
+	packBits     uint // https://github.com/Telenav/osrm-backend/blob/6283c6074066f98e6d4a9f774f21ea45407c0d52/include/extractor/packed_osm_ids.hpp#L14
 }
 
 func init() {
 	flag.StringVar(&flags.filePath, "f", "", "OSRM files(or a single file) to load, e.g. 'nevada-latest.osrm' or 'nevada-latest.osrm.nbg_nodes'. If input is 'xxx.osrm', depends on '-single_file' to load it only or load all .osrm.xxx.")
 	flag.BoolVar(&flags.singleFile, "single_file", false, "Only valid if the file path is 'xxx.osrm' from '-f'. false to load all xxx.osrm.xxx automatically, true to load the single xxx.osrm file only.")
 	flag.IntVar(&flags.printSummary, "summary", -1, "Print summary and head lines of loaded contents. <0: not print; ==0: only print summary; >0: print summary and head lines.")
+	flag.UintVar(&flags.packBits, "packed_bits", 63, "Bits for parsing packed_vector PackedOSMIDs, range [1,64]. It's 33 bits in osrm/osrm-backend, which Telenav/osrm-backend uses 63 bits instead.")
 }

integration/cmd/osrm-files-extractor/main.go

@@ -5,8 +5,9 @@ import (
 	"strings"
 
 	"github.com/Telenav/osrm-backend/integration/osrmfiles"
+	"github.com/Telenav/osrm-backend/integration/osrmfiles/dotnbgnodes"
 	"github.com/Telenav/osrm-backend/integration/osrmfiles/dotosrm"
-	"github.com/Telenav/osrm-backend/integration/osrmfiles/dotosrmdottimestamp"
+	"github.com/Telenav/osrm-backend/integration/osrmfiles/dottimestamp"
 
 	"github.com/golang/glog"
 )
@@ -16,14 +17,18 @@ const (
 
 	dotTimestampSuffix        = ".timestamp"
 	dotOSRMDotTimestampSuffix = dotOSRMSuffix + dotTimestampSuffix
+
+	dotNBGNodesSuffix        = ".nbg_nodes"
+	dotOSRMDotNBGNodesSuffix = dotOSRMSuffix + dotNBGNodesSuffix
 )
 
 // osrmBasefilePath should be 'xxx.osrm'
 func createEmptyOSRMFilesContents(osrmBasefilePath string) map[string]osrmfiles.ContentsOperator {
 
 	m := map[string]osrmfiles.ContentsOperator{}
 	m[dotOSRMSuffix] = dotosrm.New(osrmBasefilePath)
-	m[dotOSRMDotTimestampSuffix] = dotosrmdottimestamp.New(osrmBasefilePath + dotTimestampSuffix)
+	m[dotOSRMDotTimestampSuffix] = dottimestamp.New(osrmBasefilePath + dotTimestampSuffix)
+	m[dotOSRMDotNBGNodesSuffix] = dotnbgnodes.New(osrmBasefilePath+dotNBGNodesSuffix, flags.packBits)
 
 	return m
 }

integration/osrmfiles/contents_interface.go

@@ -11,6 +11,9 @@ type ContentsOperator interface {
 	// Validate checks whether the contents valid or not.
 	Validate() error
 
+	// PostProcess post process the conents once contents loaded if necessary.
+	PostProcess() error
+
 	// FindWriter find io.Writer for the specified name, contents can be filled in by the found io.Writer.
 	FindWriter(name string) (io.Writer, bool)
 

integration/osrmfiles/dotnbgnodes/contents.go

@@ -0,0 +1,92 @@
+package dotnbgnodes
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/Telenav/osrm-backend/integration/osrmfiles/fingerprint"
+	"github.com/Telenav/osrm-backend/integration/osrmfiles/meta"
+	"github.com/Telenav/osrm-backend/integration/osrmfiles/osrmtype"
+	"github.com/Telenav/osrm-backend/integration/osrmfiles/osrmtype/packed"
+	"github.com/golang/glog"
+)
+
+// Contents represents `.osrm.nbg_nodes` file structure.
+type Contents struct {
+	Fingerprint     fingerprint.Fingerprint
+	CoordinatesMeta meta.Num
+	Coordinates     osrmtype.Coordinates
+	OSMNodeIDs      packed.Uint64Vector
+
+	// for internal implementation
+	writers  map[string]io.Writer
+	filePath string
+}
+
+// New creates an empty Contents for `.osrm.nbg_nodes`.
+func New(file string, packedBits uint) *Contents {
+	c := Contents{
+		OSMNodeIDs: packed.NewUint64Vector(packedBits),
+	}
+
+	c.filePath = file
+
+	// init writers
+	c.writers = map[string]io.Writer{}
+	c.writers["osrm_fingerprint.meta"] = &c.Fingerprint
+	c.writers["/common/nbn_data/coordinates.meta"] = &c.CoordinatesMeta
+	c.writers["/common/nbn_data/coordinates"] = &c.Coordinates
+	c.writers["/common/nbn_data/osm_node_ids/number_of_elements.meta"] = &c.OSMNodeIDs.NumOfElements
+	c.writers["/common/nbn_data/osm_node_ids/packed.meta"] = &c.OSMNodeIDs.PackedMeta
+	c.writers["/common/nbn_data/osm_node_ids/packed"] = &c.OSMNodeIDs
+
+	return &c
+}
+
+// PrintSummary prints summary and head lines of contents.
+func (c *Contents) PrintSummary(head int) {
+	glog.Infof("Loaded from %s\n", c.filePath)
+	glog.Infof("  %s\n", &c.Fingerprint)
+
+	glog.Infof("  coordinates meta %d count\n", c.CoordinatesMeta)
+	for i := 0; i < head && i < len(c.Coordinates); i++ {
+		glog.Infof("    coordinate[%d] %v", i, c.Coordinates[i])
+	}
+
+	glog.Infof("  osm_node_ids number_of_elements meta %d count\n", c.OSMNodeIDs.NumOfElements)
+	glog.Infof("  osm_node_ids packed meta %d count\n", c.OSMNodeIDs.PackedMeta)
+	for i := 0; i < head && i < len(c.OSMNodeIDs.Values); i++ {
+		glog.Infof("    osm_node_ids[%d] %v", i, c.OSMNodeIDs.Values[i])
+	}
+}
+
+// Validate checks whether the contents valid or not.
+func (c *Contents) Validate() error {
+	if !c.Fingerprint.IsValid() {
+		return fmt.Errorf("invalid fingerprint %v", c.Fingerprint)
+	}
+	if uint64(c.CoordinatesMeta) != uint64(len(c.Coordinates)) {
+		return fmt.Errorf("coordinates meta not match, count in meta %d, but actual coordinates count %d", c.CoordinatesMeta, len(c.Coordinates))
+	}
+	if err := c.OSMNodeIDs.Validate(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// PostProcess post process the conents once contents loaded if necessary.
+func (c *Contents) PostProcess() error {
+	return c.OSMNodeIDs.Prune()
+}
+
+// FindWriter find io.Writer for the specified name.
+func (c *Contents) FindWriter(name string) (io.Writer, bool) {
+	w, b := c.writers[name]
+	return w, b
+}
+
+// FilePath returns the file path that stores the contents.
+func (c *Contents) FilePath() string {
+	return c.filePath
+}

integration/osrmfiles/dotosrm/contents.go

@@ -7,7 +7,7 @@ import (
 	"github.com/Telenav/osrm-backend/integration/osrmfiles/osrmtype"
 
 	"github.com/Telenav/osrm-backend/integration/osrmfiles/meta"
-	"github.com/Telenav/osrm-backend/integration/osrmfiles/querynode"
+	"github.com/Telenav/osrm-backend/integration/osrmfiles/osrmtype/querynode"
 
 	"github.com/Telenav/osrm-backend/integration/osrmfiles/fingerprint"
 	"github.com/golang/glog"
@@ -129,6 +129,11 @@ func (c *Contents) Validate() error {
 	return nil
 }
 
+// PostProcess post process the conents once contents loaded if necessary.
+func (c *Contents) PostProcess() error {
+	return nil // nothing need to do
+}
+
 // FindWriter find io.Writer for the specified name.
 func (c *Contents) FindWriter(name string) (io.Writer, bool) {
 	w, b := c.writers[name]

integration/osrmfiles/dotosrmdottimestamp/contents.go → integration/osrmfiles/dottimestamp/contents.go

@@ -1,4 +1,4 @@
-package dotosrmdottimestamp
+package dottimestamp
 
 import (
 	"bytes"
@@ -47,6 +47,11 @@ func (c *Contents) Validate() error {
 	return nil
 }
 
+// PostProcess post process the conents once contents loaded if necessary.
+func (c *Contents) PostProcess() error {
+	return nil // nothing need to do
+}
+
 // PrintSummary prints summary and head lines of contents.
 func (c *Contents) PrintSummary(head int) {
 	glog.Infof("Loaded from %s\n", c.filePath)

integration/osrmfiles/osrmtype/coordinate.go

@@ -0,0 +1,51 @@
+package osrmtype
+
+import "encoding/binary"
+
+// FixedLat defines FixedLatitude with COORDINATE_PRECISION = 1e6
+// C++ Implementation: https://github.com/Telenav/osrm-backend/blob/6283c6074066f98e6d4a9f774f21ea45407c0d52/include/util/coordinate.hpp#L66
+// C++ implementation toFixed()/toFloat() see https://github.com/Telenav/osrm-backend/blob/6283c6074066f98e6d4a9f774f21ea45407c0d52/include/util/coordinate.hpp#L88
+type FixedLat int32
+
+// FixedLon defines FixedLatitude with COORDINATE_PRECISION = 1e6
+// C++ Implementation: https://github.com/Telenav/osrm-backend/blob/6283c6074066f98e6d4a9f774f21ea45407c0d52/include/util/coordinate.hpp#L66
+// C++ implementation toFixed()/toFloat() see https://github.com/Telenav/osrm-backend/blob/6283c6074066f98e6d4a9f774f21ea45407c0d52/include/util/coordinate.hpp#L88
+type FixedLon int32
+
+// Coordinate represents Coordinate structure.
+// C++ implementation: https://github.com/Telenav/osrm-backend/blob/6283c6074066f98e6d4a9f774f21ea45407c0d52/include/util/coordinate.hpp#L185
+type Coordinate struct {
+	FixedLon
+	FixedLat
+}
+
+// Coordinates represents Coordinate vector structure.
+type Coordinates []Coordinate
+
+const (
+	fixedLonBytes   = 4 // int32
+	fixedLatBytes   = 4 // int32
+	coordinateBytes = fixedLonBytes + fixedLatBytes
+)
+
+func (c *Coordinates) Write(p []byte) (int, error) {
+
+	var writeLen int
+	writeP := p
+	for {
+		if len(writeP) < coordinateBytes {
+			break
+		}
+
+		var coordinate Coordinate
+		coordinate.FixedLon = FixedLon(binary.LittleEndian.Uint32(writeP))
+		coordinate.FixedLat = FixedLat(binary.LittleEndian.Uint32(writeP[fixedLonBytes:]))
+
+		*c = append(*c, coordinate)
+
+		writeP = writeP[coordinateBytes:]
+		writeLen += coordinateBytes
+	}
+
+	return writeLen, nil
+}

integration/osrmfiles/osrmtype/coordinate_test.go

@@ -0,0 +1,64 @@
+package osrmtype
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestWriteCoordinates(t *testing.T) {
+	cases := []struct {
+		p []byte
+		Coordinates
+	}{
+		{
+			[]byte{
+				0x19, 0xc4, 0xd8, 0xf8, 0x8c, 0xdb, 0x59, 0x02,
+			},
+			Coordinates{
+				Coordinate{FixedLon: -120011751, FixedLat: 39443340},
+			},
+		},
+		{
+			[]byte{
+				0x19, 0xc4, 0xd8, 0xf8, 0x8c, 0xdb, 0x59, 0x02,
+				0xFF, // redundant byte
+			},
+			Coordinates{
+				Coordinate{FixedLon: -120011751, FixedLat: 39443340},
+			},
+		},
+		{
+			[]byte{
+				0x19, 0xc4, 0xd8, 0xf8, 0x8c, 0xdb, 0x59, 0x02,
+				0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // redundant bytes
+			},
+			Coordinates{
+				Coordinate{FixedLon: -120011751, FixedLat: 39443340},
+			},
+		},
+		{
+			[]byte{
+				0x19, 0xc4, 0xd8, 0xf8, 0x8c, 0xdb, 0x59, 0x02,
+				0x66, 0x03, 0x24, 0xf9, 0x6b, 0xba, 0x27, 0x02,
+			},
+			Coordinates{
+				Coordinate{FixedLon: -120011751, FixedLat: 39443340},
+				Coordinate{FixedLon: -115080346, FixedLat: 36158059},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		coordinates := Coordinates{}
+		writeLen, err := coordinates.Write(c.p)
+		if err != nil {
+			t.Error(err)
+		}
+		if len(c.p)-writeLen != len(c.p)%coordinateBytes {
+			t.Errorf("len(p) %d but write len %d", len(c.p), writeLen)
+		}
+		if !reflect.DeepEqual(coordinates, c.Coordinates) {
+			t.Errorf("parse %v expect %v but got %v", c.p, c.Coordinates, coordinates)
+		}
+	}
+}