Skip to content

Commit

Permalink
Feature/observe dot timestamp (#138)
Browse files Browse the repository at this point in the history
* feat: load .osrm.timestamp file

* refactor: load .osrm[.xxx] interfaces and process

* docs: sample for .osrm.timestamp load
  • Loading branch information
wangyoucao577 authored Jan 23, 2020
1 parent e6350eb commit 037a545
Show file tree
Hide file tree
Showing 7 changed files with 268 additions and 99 deletions.
68 changes: 36 additions & 32 deletions integration/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,36 +68,40 @@ $ tar tvf nevada-latest.osrm
$
$ # we can use `osrm-files-extractor` to see its details.
$ ./osrm-files-extractor -alsologtostderr -f nevada-latest.osrm -summary 5
I1218 14:12:25.868053 1747 contents.go:81] Loaded from nevada-latest.osrm
I1218 14:12:25.869828 1747 contents.go:82] OSRN v5.22.0
I1218 14:12:25.869848 1747 contents.go:84] nodes meta 1142236 count 1142236
I1218 14:12:25.869853 1747 contents.go:86] node[0] {-120011751 39443340 26798725}
I1218 14:12:25.870092 1747 contents.go:86] node[1] {-120017543 39440794 26798726}
I1218 14:12:25.870097 1747 contents.go:86] node[2] {-120031913 39431791 26798727}
I1218 14:12:25.870100 1747 contents.go:86] node[3] {-120035895 39423872 26798728}
I1218 14:12:25.870103 1747 contents.go:86] node[4] {-120030001 39416698 26798729}
I1218 14:12:25.870106 1747 contents.go:89] barriers meta 152 count 152
I1218 14:12:25.870110 1747 contents.go:91] barrier[0] 66546
I1218 14:12:25.870114 1747 contents.go:91] barrier[1] 196332
I1218 14:12:25.870117 1747 contents.go:91] barrier[2] 235061
I1218 14:12:25.870122 1747 contents.go:91] barrier[3] 316127
I1218 14:12:25.870125 1747 contents.go:91] barrier[4] 335454
I1218 14:12:25.870128 1747 contents.go:94] traffic_lights meta 4036 count 4036
I1218 14:12:25.870132 1747 contents.go:96] traffic_lights[0] 6467
I1218 14:12:25.870135 1747 contents.go:96] traffic_lights[1] 6479
I1218 14:12:25.870138 1747 contents.go:96] traffic_lights[2] 6483
I1218 14:12:25.870140 1747 contents.go:96] traffic_lights[3] 13948
I1218 14:12:25.870143 1747 contents.go:96] traffic_lights[4] 13981
I1218 14:12:25.870146 1747 contents.go:99] edges meta 1234521 count 1234521
I1218 14:12:25.870150 1747 contents.go:101] edges[0] {0 487556 24 24 54.78657 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}}
I1218 14:12:25.870844 1747 contents.go:101] edges[1] {487557 0 21 21 49.09447 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}}
I1218 14:12:25.870853 1747 contents.go:101] edges[2] {487554 1 50 50 116.59047 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}}
I1218 14:12:25.870860 1747 contents.go:101] edges[3] {1 604105 23 23 53.611294 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}}
I1218 14:12:25.870866 1747 contents.go:101] edges[4] {487542 2 54 54 126.03433 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}}
I1218 14:12:25.870872 1747 contents.go:104] annotations meta 169386 count 169386
I1218 14:12:25.870876 1747 contents.go:106] annotations[0] {5 65535 1 1 false}
I1218 14:12:25.870884 1747 contents.go:106] annotations[1] {0 65535 0 1 false}
I1218 14:12:25.870888 1747 contents.go:106] annotations[2] {10 65535 0 1 false}
I1218 14:12:25.870892 1747 contents.go:106] annotations[3] {15 65535 0 1 false}
I1218 14:12:25.870896 1747 contents.go:106] annotations[4] {0 65535 0 1 false}
I0121 15:01:55.282673 7124 contents.go:60] Loaded from nevada-latest.osrm
I0121 15:01:55.285059 7124 contents.go:61] OSRN v5.22.0
I0121 15:01:55.285161 7124 contents.go:63] nodes meta 1092695 count 1092695
I0121 15:01:55.285172 7124 contents.go:65] node[0] {-120011751 39443340 26798725}
I0121 15:01:55.285192 7124 contents.go:65] node[1] {-120017543 39440794 26798726}
I0121 15:01:55.285202 7124 contents.go:65] node[2] {-120031913 39431791 26798727}
I0121 15:01:55.285228 7124 contents.go:65] node[3] {-120035895 39423872 26798728}
I0121 15:01:55.285256 7124 contents.go:65] node[4] {-120030001 39416698 26798729}
I0121 15:01:55.285266 7124 contents.go:68] barriers meta 85 count 85
I0121 15:01:55.285292 7124 contents.go:70] barrier[0] 68738
I0121 15:01:55.285321 7124 contents.go:70] barrier[1] 127356
I0121 15:01:55.285338 7124 contents.go:70] barrier[2] 205601
I0121 15:01:55.285348 7124 contents.go:70] barrier[3] 331924
I0121 15:01:55.285410 7124 contents.go:70] barrier[4] 351581
I0121 15:01:55.285439 7124 contents.go:73] traffic_lights meta 3821 count 3821
I0121 15:01:55.285466 7124 contents.go:75] traffic_lights[0] 6584
I0121 15:01:55.285493 7124 contents.go:75] traffic_lights[1] 6596
I0121 15:01:55.285524 7124 contents.go:75] traffic_lights[2] 6600
I0121 15:01:55.285548 7124 contents.go:75] traffic_lights[3] 15405
I0121 15:01:55.285576 7124 contents.go:75] traffic_lights[4] 15439
I0121 15:01:55.285600 7124 contents.go:78] edges meta 1179277 count 1179277
I0121 15:01:55.285650 7124 contents.go:80] edges[0] {0 503728 24 24 54.78657 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}}
I0121 15:01:55.285726 7124 contents.go:80] edges[1] {503729 0 21 21 49.09447 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}}
I0121 15:01:55.285753 7124 contents.go:80] edges[2] {503726 1 50 50 116.59047 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}}
I0121 15:01:55.285785 7124 contents.go:80] edges[3] {1 618170 23 23 53.611294 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}}
I0121 15:01:55.285814 7124 contents.go:80] edges[4] {503714 2 54 54 126.03433 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}}
I0121 15:01:55.285840 7124 contents.go:83] annotations meta 158342 count 158342
I0121 15:01:55.285851 7124 contents.go:85] annotations[0] {5 65535 1 1 false}
I0121 15:01:55.285899 7124 contents.go:85] annotations[1] {0 65535 0 1 false}
I0121 15:01:55.285924 7124 contents.go:85] annotations[2] {10 65535 0 1 false}
I0121 15:01:55.285951 7124 contents.go:85] annotations[3] {0 65535 0 1 false}
I0121 15:01:55.286009 7124 contents.go:85] annotations[4] {15 65535 0 1 false}
I0121 15:01:55.286492 7124 contents.go:52] Loaded from nevada-latest.osrm.timestamp
I0121 15:01:55.286525 7124 contents.go:53] OSRN v5.22.0
I0121 15:01:55.286539 7124 contents.go:55] timestamp(a.k.a. data_version) meta 20 count 20
I0121 15:01:55.286549 7124 contents.go:57] timestamp(a.k.a. data_version) 2019-01-24T21:15:02Z
```
4 changes: 3 additions & 1 deletion integration/cmd/osrm-files-extractor/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ import "flag"

var flags struct {
filePath string
singleFile bool
printSummary int
}

func init() {
flag.StringVar(&flags.filePath, "f", "", "Single OSRM file to load, e.g. 'nevada-latest.osrm' or 'nevada-latest.osrm.nbg_nodes'.")
flag.StringVar(&flags.filePath, "f", "", "OSRM files(or a single file) to load, e.g. 'nevada-latest.osrm' or 'nevada-latest.osrm.nbg_nodes'. If input is 'xxx.osrm', depends on '-single_file' to load it only or load all .osrm.xxx.")
flag.BoolVar(&flags.singleFile, "single_file", false, "Only valid if the file path is 'xxx.osrm' from '-f'. false to load all xxx.osrm.xxx automatically, true to load the single xxx.osrm file only.")
flag.IntVar(&flags.printSummary, "summary", -1, "Print summary and head lines of loaded contents. <0: not print; ==0: only print summary; >0: print summary and head lines.")
}
64 changes: 59 additions & 5 deletions integration/cmd/osrm-files-extractor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,75 @@ package main

import (
"flag"
"strings"

"github.com/Telenav/osrm-backend/integration/osrmfiles"
"github.com/Telenav/osrm-backend/integration/osrmfiles/dotosrm"
"github.com/Telenav/osrm-backend/integration/osrmfiles/dotosrmdottimestamp"

"github.com/golang/glog"
)

const (
dotOSRMSuffix = ".osrm"

dotTimestampSuffix = ".timestamp"
dotOSRMDotTimestampSuffix = dotOSRMSuffix + dotTimestampSuffix
)

// osrmBasefilePath should be 'xxx.osrm'
func createEmptyOSRMFilesContents(osrmBasefilePath string) map[string]osrmfiles.ContentsOperator {

m := map[string]osrmfiles.ContentsOperator{}
m[dotOSRMSuffix] = dotosrm.New(osrmBasefilePath)
m[dotOSRMDotTimestampSuffix] = dotosrmdottimestamp.New(osrmBasefilePath + dotTimestampSuffix)

return m
}

func main() {
flag.Parse()
defer glog.Flush()

contents, err := dotosrm.Load(flags.filePath)
if err != nil {
glog.Error(err)
suffixIndex := strings.LastIndex(flags.filePath, dotOSRMSuffix)
if suffixIndex < 0 {
glog.Errorf("file path %s should end by .osrm[.xxx]\n", flags.filePath)
return
}
if flags.printSummary >= 0 {
contents.PrintSummary(flags.printSummary)
suffix := flags.filePath[suffixIndex:] // should be '.osrm' or '.osrm.xxx'
baseFilePath := flags.filePath[:suffixIndex] + dotOSRMSuffix // should be xxx.osrm

// create empty files and contents mapping for loading later
osrmContents := createEmptyOSRMFilesContents(baseFilePath)

if suffix != dotOSRMSuffix || (suffix == dotOSRMSuffix && flags.singleFile) {
// only keep the specified contents if want to load single file
for k := range osrmContents {
if k != suffix {
delete(osrmContents, k)
}
}
}

if len(osrmContents) == 0 {
glog.Warningf("nothing need to load for %s", flags.filePath)
return
}

// load contents and print summary
for k, c := range osrmContents {
if c == nil {
glog.Errorf("nil Contents to load %s", k)
continue
}
if err := osrmfiles.Load(c); err != nil {
glog.Error(err)
continue
}

if flags.printSummary >= 0 {
c.PrintSummary(flags.printSummary)
}
}

}
19 changes: 19 additions & 0 deletions integration/osrmfiles/contents_interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package osrmfiles

import "io"

// ContentsOperator is the interfaces that wraps Contents' methods.
type ContentsOperator interface {

// PrintSummary prints summary and head lines of contents.
PrintSummary(head int)

// Validate checks whether the contents valid or not.
Validate() error

// FindWriter find io.Writer for the specified name, contents can be filled in by the found io.Writer.
FindWriter(name string) (io.Writer, bool)

// FilePath returns the file path that stores the contents.
FilePath() string
}
93 changes: 32 additions & 61 deletions integration/osrmfiles/dotosrm/contents.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package dotosrm

import (
"archive/tar"
"fmt"
"io"
"os"

"github.com/Telenav/osrm-backend/integration/osrmfiles/osrmtype"

Expand Down Expand Up @@ -34,49 +32,30 @@ type Contents struct {
filePath string
}

// Load `.osrm` file to generate a new contents structure.
func Load(file string) (*Contents, error) {
f, err := os.Open(file)
defer f.Close()
if err != nil {
return nil, err
}
glog.V(2).Infof("open %s succeed.\n", file)

contents := new()

// Open and iterate through the files in the archive.
tr := tar.NewReader(f)
for {
hdr, err := tr.Next()
if err == io.EOF {
break // End of archive
}
if err != nil {
glog.Fatal(err)
}
glog.V(1).Infof("%s\n", hdr.Name)
writer, found := contents.writers[hdr.Name]
if !found {
glog.Warningf("unrecognized content in tar: %s", hdr.Name)
continue
}
// New creates an empty Contents for `.osrm`.
func New(file string) *Contents {
c := Contents{}

if _, err := io.Copy(writer, tr); err != nil {
glog.Fatal(err)
}
}
c.filePath = file

// validate loaded contents
if err := contents.validate(); err != nil {
return nil, err
}
// init writers
c.writers = map[string]io.Writer{}
c.writers["osrm_fingerprint.meta"] = &c.Fingerprint
c.writers["/extractor/nodes.meta"] = &c.NodesMeta
c.writers["/extractor/nodes"] = &c.Nodes
c.writers["/extractor/barriers.meta"] = &c.BarriersMeta
c.writers["/extractor/barriers"] = &c.Barriers
c.writers["/extractor/traffic_lights.meta"] = &c.TrafficLightsMeta
c.writers["/extractor/traffic_lights"] = &c.TrafficLights
c.writers["/extractor/edges.meta"] = &c.EdgesMeta
c.writers["/extractor/edges"] = &c.Edges
c.writers["/extractor/annotations.meta"] = &c.AnnotationsMeta
c.writers["/extractor/annotations"] = &c.Annotations

contents.filePath = file
return contents, nil
return &c
}

// PrintSummary prints summary and head lines of current contents.
// PrintSummary prints summary and head lines of contents.
func (c *Contents) PrintSummary(head int) {
glog.Infof("Loaded from %s\n", c.filePath)
glog.Infof(" %s\n", &c.Fingerprint)
Expand Down Expand Up @@ -108,27 +87,8 @@ func (c *Contents) PrintSummary(head int) {

}

func new() *Contents {
c := Contents{}

// init writers
c.writers = map[string]io.Writer{}
c.writers["osrm_fingerprint.meta"] = &c.Fingerprint
c.writers["/extractor/nodes.meta"] = &c.NodesMeta
c.writers["/extractor/nodes"] = &c.Nodes
c.writers["/extractor/barriers.meta"] = &c.BarriersMeta
c.writers["/extractor/barriers"] = &c.Barriers
c.writers["/extractor/traffic_lights.meta"] = &c.TrafficLightsMeta
c.writers["/extractor/traffic_lights"] = &c.TrafficLights
c.writers["/extractor/edges.meta"] = &c.EdgesMeta
c.writers["/extractor/edges"] = &c.Edges
c.writers["/extractor/annotations.meta"] = &c.AnnotationsMeta
c.writers["/extractor/annotations"] = &c.Annotations

return &c
}

func (c *Contents) validate() error {
// Validate checks whether the contents valid or not.
func (c *Contents) Validate() error {
if !c.Fingerprint.IsValid() {
return fmt.Errorf("invalid fingerprint %v", c.Fingerprint)
}
Expand Down Expand Up @@ -168,3 +128,14 @@ func (c *Contents) validate() error {

return nil
}

// FindWriter find io.Writer for the specified name.
func (c *Contents) FindWriter(name string) (io.Writer, bool) {
w, b := c.writers[name]
return w, b
}

// FilePath returns the file path that stores the contents.
func (c *Contents) FilePath() string {
return c.filePath
}
70 changes: 70 additions & 0 deletions integration/osrmfiles/dotosrmdottimestamp/contents.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package dotosrmdottimestamp

import (
"bytes"
"fmt"
"io"

"github.com/Telenav/osrm-backend/integration/osrmfiles/fingerprint"
"github.com/Telenav/osrm-backend/integration/osrmfiles/meta"
"github.com/golang/glog"
)

// Contents represents `.osrm.timestamp` file structure.
type Contents struct {
Fingerprint fingerprint.Fingerprint
TimestampMeta meta.Num
Timestamp bytes.Buffer

// for internal implementation
writers map[string]io.Writer
filePath string
}

// New creates an empty Contents for `.osrm.timestamp`.
func New(file string) *Contents {
c := Contents{}

c.filePath = file

// init writers
c.writers = map[string]io.Writer{}
c.writers["osrm_fingerprint.meta"] = &c.Fingerprint
c.writers["/common/timestamp.meta"] = &c.TimestampMeta
c.writers["/common/timestamp"] = &c.Timestamp

return &c
}

// Validate checks whether the contents valid or not.
func (c *Contents) Validate() error {
if !c.Fingerprint.IsValid() {
return fmt.Errorf("invalid fingerprint %v", c.Fingerprint)
}
if uint64(c.TimestampMeta) != uint64(c.Timestamp.Len()) {
return fmt.Errorf("timestamp meta not match, count in meta %d, but actual timestamp bytse count %d", c.TimestampMeta, c.Timestamp.Len())
}
return nil
}

// PrintSummary prints summary and head lines of contents.
func (c *Contents) PrintSummary(head int) {
glog.Infof("Loaded from %s\n", c.filePath)
glog.Infof(" %s\n", &c.Fingerprint)

glog.Infof(" timestamp(a.k.a. data_version) meta %d count %d\n", c.TimestampMeta, c.Timestamp.Len())
if head > 0 {
glog.Infof(" timestamp(a.k.a. data_version) %v\n", c.Timestamp.String())
}
}

// FindWriter find io.Writer for the specified name.
func (c *Contents) FindWriter(name string) (io.Writer, bool) {
w, b := c.writers[name]
return w, b
}

// FilePath returns the file path that stores the contents.
func (c *Contents) FilePath() string {
return c.filePath
}
Loading

0 comments on commit 037a545

Please sign in to comment.