From 037a545659cef519f976360ae5c90dffcbafc145 Mon Sep 17 00:00:00 2001 From: Jay Date: Thu, 23 Jan 2020 09:18:45 +0800 Subject: [PATCH] Feature/observe dot timestamp (#138) * feat: load .osrm.timestamp file * refactor: load .osrm[.xxx] interfaces and process * docs: sample for .osrm.timestamp load --- integration/README.md | 68 +++++++------- integration/cmd/osrm-files-extractor/flags.go | 4 +- integration/cmd/osrm-files-extractor/main.go | 64 ++++++++++++- integration/osrmfiles/contents_interface.go | 19 ++++ integration/osrmfiles/dotosrm/contents.go | 93 +++++++------------ .../osrmfiles/dotosrmdottimestamp/contents.go | 70 ++++++++++++++ integration/osrmfiles/tar_loader.go | 49 ++++++++++ 7 files changed, 268 insertions(+), 99 deletions(-) create mode 100644 integration/osrmfiles/contents_interface.go create mode 100644 integration/osrmfiles/dotosrmdottimestamp/contents.go create mode 100644 integration/osrmfiles/tar_loader.go diff --git a/integration/README.md b/integration/README.md index 67e9586acd7..6bddf5c40ec 100644 --- a/integration/README.md +++ b/integration/README.md @@ -68,36 +68,40 @@ $ tar tvf nevada-latest.osrm $ $ # we can use `osrm-files-extractor` to see its details. $ ./osrm-files-extractor -alsologtostderr -f nevada-latest.osrm -summary 5 -I1218 14:12:25.868053 1747 contents.go:81] Loaded from nevada-latest.osrm -I1218 14:12:25.869828 1747 contents.go:82] OSRN v5.22.0 -I1218 14:12:25.869848 1747 contents.go:84] nodes meta 1142236 count 1142236 -I1218 14:12:25.869853 1747 contents.go:86] node[0] {-120011751 39443340 26798725} -I1218 14:12:25.870092 1747 contents.go:86] node[1] {-120017543 39440794 26798726} -I1218 14:12:25.870097 1747 contents.go:86] node[2] {-120031913 39431791 26798727} -I1218 14:12:25.870100 1747 contents.go:86] node[3] {-120035895 39423872 26798728} -I1218 14:12:25.870103 1747 contents.go:86] node[4] {-120030001 39416698 26798729} -I1218 14:12:25.870106 1747 contents.go:89] barriers meta 152 count 152 -I1218 14:12:25.870110 1747 contents.go:91] barrier[0] 66546 -I1218 14:12:25.870114 1747 contents.go:91] barrier[1] 196332 -I1218 14:12:25.870117 1747 contents.go:91] barrier[2] 235061 -I1218 14:12:25.870122 1747 contents.go:91] barrier[3] 316127 -I1218 14:12:25.870125 1747 contents.go:91] barrier[4] 335454 -I1218 14:12:25.870128 1747 contents.go:94] traffic_lights meta 4036 count 4036 -I1218 14:12:25.870132 1747 contents.go:96] traffic_lights[0] 6467 -I1218 14:12:25.870135 1747 contents.go:96] traffic_lights[1] 6479 -I1218 14:12:25.870138 1747 contents.go:96] traffic_lights[2] 6483 -I1218 14:12:25.870140 1747 contents.go:96] traffic_lights[3] 13948 -I1218 14:12:25.870143 1747 contents.go:96] traffic_lights[4] 13981 -I1218 14:12:25.870146 1747 contents.go:99] edges meta 1234521 count 1234521 -I1218 14:12:25.870150 1747 contents.go:101] edges[0] {0 487556 24 24 54.78657 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}} -I1218 14:12:25.870844 1747 contents.go:101] edges[1] {487557 0 21 21 49.09447 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}} -I1218 14:12:25.870853 1747 contents.go:101] edges[2] {487554 1 50 50 116.59047 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}} -I1218 14:12:25.870860 1747 contents.go:101] edges[3] {1 604105 23 23 53.611294 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}} -I1218 14:12:25.870866 1747 contents.go:101] edges[4] {487542 2 54 54 126.03433 {2147483647 false} 44494 {true false false false false true false {true false false 0 2} 0 0}} -I1218 14:12:25.870872 1747 contents.go:104] annotations meta 169386 count 169386 -I1218 14:12:25.870876 1747 contents.go:106] annotations[0] {5 65535 1 1 false} -I1218 14:12:25.870884 1747 contents.go:106] annotations[1] {0 65535 0 1 false} -I1218 14:12:25.870888 1747 contents.go:106] annotations[2] {10 65535 0 1 false} -I1218 14:12:25.870892 1747 contents.go:106] annotations[3] {15 65535 0 1 false} -I1218 14:12:25.870896 1747 contents.go:106] annotations[4] {0 65535 0 1 false} +I0121 15:01:55.282673 7124 contents.go:60] Loaded from nevada-latest.osrm +I0121 15:01:55.285059 7124 contents.go:61] OSRN v5.22.0 +I0121 15:01:55.285161 7124 contents.go:63] nodes meta 1092695 count 1092695 +I0121 15:01:55.285172 7124 contents.go:65] node[0] {-120011751 39443340 26798725} +I0121 15:01:55.285192 7124 contents.go:65] node[1] {-120017543 39440794 26798726} +I0121 15:01:55.285202 7124 contents.go:65] node[2] {-120031913 39431791 26798727} +I0121 15:01:55.285228 7124 contents.go:65] node[3] {-120035895 39423872 26798728} +I0121 15:01:55.285256 7124 contents.go:65] node[4] {-120030001 39416698 26798729} +I0121 15:01:55.285266 7124 contents.go:68] barriers meta 85 count 85 +I0121 15:01:55.285292 7124 contents.go:70] barrier[0] 68738 +I0121 15:01:55.285321 7124 contents.go:70] barrier[1] 127356 +I0121 15:01:55.285338 7124 contents.go:70] barrier[2] 205601 +I0121 15:01:55.285348 7124 contents.go:70] barrier[3] 331924 +I0121 15:01:55.285410 7124 contents.go:70] barrier[4] 351581 +I0121 15:01:55.285439 7124 contents.go:73] traffic_lights meta 3821 count 3821 +I0121 15:01:55.285466 7124 contents.go:75] traffic_lights[0] 6584 +I0121 15:01:55.285493 7124 contents.go:75] traffic_lights[1] 6596 +I0121 15:01:55.285524 7124 contents.go:75] traffic_lights[2] 6600 +I0121 15:01:55.285548 7124 contents.go:75] traffic_lights[3] 15405 +I0121 15:01:55.285576 7124 contents.go:75] traffic_lights[4] 15439 +I0121 15:01:55.285600 7124 contents.go:78] edges meta 1179277 count 1179277 +I0121 15:01:55.285650 7124 contents.go:80] edges[0] {0 503728 24 24 54.78657 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}} +I0121 15:01:55.285726 7124 contents.go:80] edges[1] {503729 0 21 21 49.09447 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}} +I0121 15:01:55.285753 7124 contents.go:80] edges[2] {503726 1 50 50 116.59047 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}} +I0121 15:01:55.285785 7124 contents.go:80] edges[3] {1 618170 23 23 53.611294 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}} +I0121 15:01:55.285814 7124 contents.go:80] edges[4] {503714 2 54 54 126.03433 {2147483647 false} 44793 {true false false false false true false {true false false 0 2} 0 0}} +I0121 15:01:55.285840 7124 contents.go:83] annotations meta 158342 count 158342 +I0121 15:01:55.285851 7124 contents.go:85] annotations[0] {5 65535 1 1 false} +I0121 15:01:55.285899 7124 contents.go:85] annotations[1] {0 65535 0 1 false} +I0121 15:01:55.285924 7124 contents.go:85] annotations[2] {10 65535 0 1 false} +I0121 15:01:55.285951 7124 contents.go:85] annotations[3] {0 65535 0 1 false} +I0121 15:01:55.286009 7124 contents.go:85] annotations[4] {15 65535 0 1 false} +I0121 15:01:55.286492 7124 contents.go:52] Loaded from nevada-latest.osrm.timestamp +I0121 15:01:55.286525 7124 contents.go:53] OSRN v5.22.0 +I0121 15:01:55.286539 7124 contents.go:55] timestamp(a.k.a. data_version) meta 20 count 20 +I0121 15:01:55.286549 7124 contents.go:57] timestamp(a.k.a. data_version) 2019-01-24T21:15:02Z ``` diff --git a/integration/cmd/osrm-files-extractor/flags.go b/integration/cmd/osrm-files-extractor/flags.go index 73767042122..9f7b674af80 100644 --- a/integration/cmd/osrm-files-extractor/flags.go +++ b/integration/cmd/osrm-files-extractor/flags.go @@ -4,10 +4,12 @@ import "flag" var flags struct { filePath string + singleFile bool printSummary int } func init() { - flag.StringVar(&flags.filePath, "f", "", "Single OSRM file to load, e.g. 'nevada-latest.osrm' or 'nevada-latest.osrm.nbg_nodes'.") + flag.StringVar(&flags.filePath, "f", "", "OSRM files(or a single file) to load, e.g. 'nevada-latest.osrm' or 'nevada-latest.osrm.nbg_nodes'. If input is 'xxx.osrm', depends on '-single_file' to load it only or load all .osrm.xxx.") + flag.BoolVar(&flags.singleFile, "single_file", false, "Only valid if the file path is 'xxx.osrm' from '-f'. false to load all xxx.osrm.xxx automatically, true to load the single xxx.osrm file only.") flag.IntVar(&flags.printSummary, "summary", -1, "Print summary and head lines of loaded contents. <0: not print; ==0: only print summary; >0: print summary and head lines.") } diff --git a/integration/cmd/osrm-files-extractor/main.go b/integration/cmd/osrm-files-extractor/main.go index d08c15ace75..53825be9e9d 100644 --- a/integration/cmd/osrm-files-extractor/main.go +++ b/integration/cmd/osrm-files-extractor/main.go @@ -2,21 +2,75 @@ package main import ( "flag" + "strings" + "github.com/Telenav/osrm-backend/integration/osrmfiles" "github.com/Telenav/osrm-backend/integration/osrmfiles/dotosrm" + "github.com/Telenav/osrm-backend/integration/osrmfiles/dotosrmdottimestamp" + "github.com/golang/glog" ) +const ( + dotOSRMSuffix = ".osrm" + + dotTimestampSuffix = ".timestamp" + dotOSRMDotTimestampSuffix = dotOSRMSuffix + dotTimestampSuffix +) + +// osrmBasefilePath should be 'xxx.osrm' +func createEmptyOSRMFilesContents(osrmBasefilePath string) map[string]osrmfiles.ContentsOperator { + + m := map[string]osrmfiles.ContentsOperator{} + m[dotOSRMSuffix] = dotosrm.New(osrmBasefilePath) + m[dotOSRMDotTimestampSuffix] = dotosrmdottimestamp.New(osrmBasefilePath + dotTimestampSuffix) + + return m +} + func main() { flag.Parse() defer glog.Flush() - contents, err := dotosrm.Load(flags.filePath) - if err != nil { - glog.Error(err) + suffixIndex := strings.LastIndex(flags.filePath, dotOSRMSuffix) + if suffixIndex < 0 { + glog.Errorf("file path %s should end by .osrm[.xxx]\n", flags.filePath) return } - if flags.printSummary >= 0 { - contents.PrintSummary(flags.printSummary) + suffix := flags.filePath[suffixIndex:] // should be '.osrm' or '.osrm.xxx' + baseFilePath := flags.filePath[:suffixIndex] + dotOSRMSuffix // should be xxx.osrm + + // create empty files and contents mapping for loading later + osrmContents := createEmptyOSRMFilesContents(baseFilePath) + + if suffix != dotOSRMSuffix || (suffix == dotOSRMSuffix && flags.singleFile) { + // only keep the specified contents if want to load single file + for k := range osrmContents { + if k != suffix { + delete(osrmContents, k) + } + } + } + + if len(osrmContents) == 0 { + glog.Warningf("nothing need to load for %s", flags.filePath) + return } + + // load contents and print summary + for k, c := range osrmContents { + if c == nil { + glog.Errorf("nil Contents to load %s", k) + continue + } + if err := osrmfiles.Load(c); err != nil { + glog.Error(err) + continue + } + + if flags.printSummary >= 0 { + c.PrintSummary(flags.printSummary) + } + } + } diff --git a/integration/osrmfiles/contents_interface.go b/integration/osrmfiles/contents_interface.go new file mode 100644 index 00000000000..8ff611df783 --- /dev/null +++ b/integration/osrmfiles/contents_interface.go @@ -0,0 +1,19 @@ +package osrmfiles + +import "io" + +// ContentsOperator is the interfaces that wraps Contents' methods. +type ContentsOperator interface { + + // PrintSummary prints summary and head lines of contents. + PrintSummary(head int) + + // Validate checks whether the contents valid or not. + Validate() error + + // FindWriter find io.Writer for the specified name, contents can be filled in by the found io.Writer. + FindWriter(name string) (io.Writer, bool) + + // FilePath returns the file path that stores the contents. + FilePath() string +} diff --git a/integration/osrmfiles/dotosrm/contents.go b/integration/osrmfiles/dotosrm/contents.go index 4812280a943..4d72779ec0a 100644 --- a/integration/osrmfiles/dotosrm/contents.go +++ b/integration/osrmfiles/dotosrm/contents.go @@ -1,10 +1,8 @@ package dotosrm import ( - "archive/tar" "fmt" "io" - "os" "github.com/Telenav/osrm-backend/integration/osrmfiles/osrmtype" @@ -34,49 +32,30 @@ type Contents struct { filePath string } -// Load `.osrm` file to generate a new contents structure. -func Load(file string) (*Contents, error) { - f, err := os.Open(file) - defer f.Close() - if err != nil { - return nil, err - } - glog.V(2).Infof("open %s succeed.\n", file) - - contents := new() - - // Open and iterate through the files in the archive. - tr := tar.NewReader(f) - for { - hdr, err := tr.Next() - if err == io.EOF { - break // End of archive - } - if err != nil { - glog.Fatal(err) - } - glog.V(1).Infof("%s\n", hdr.Name) - writer, found := contents.writers[hdr.Name] - if !found { - glog.Warningf("unrecognized content in tar: %s", hdr.Name) - continue - } +// New creates an empty Contents for `.osrm`. +func New(file string) *Contents { + c := Contents{} - if _, err := io.Copy(writer, tr); err != nil { - glog.Fatal(err) - } - } + c.filePath = file - // validate loaded contents - if err := contents.validate(); err != nil { - return nil, err - } + // init writers + c.writers = map[string]io.Writer{} + c.writers["osrm_fingerprint.meta"] = &c.Fingerprint + c.writers["/extractor/nodes.meta"] = &c.NodesMeta + c.writers["/extractor/nodes"] = &c.Nodes + c.writers["/extractor/barriers.meta"] = &c.BarriersMeta + c.writers["/extractor/barriers"] = &c.Barriers + c.writers["/extractor/traffic_lights.meta"] = &c.TrafficLightsMeta + c.writers["/extractor/traffic_lights"] = &c.TrafficLights + c.writers["/extractor/edges.meta"] = &c.EdgesMeta + c.writers["/extractor/edges"] = &c.Edges + c.writers["/extractor/annotations.meta"] = &c.AnnotationsMeta + c.writers["/extractor/annotations"] = &c.Annotations - contents.filePath = file - return contents, nil + return &c } -// PrintSummary prints summary and head lines of current contents. +// PrintSummary prints summary and head lines of contents. func (c *Contents) PrintSummary(head int) { glog.Infof("Loaded from %s\n", c.filePath) glog.Infof(" %s\n", &c.Fingerprint) @@ -108,27 +87,8 @@ func (c *Contents) PrintSummary(head int) { } -func new() *Contents { - c := Contents{} - - // init writers - c.writers = map[string]io.Writer{} - c.writers["osrm_fingerprint.meta"] = &c.Fingerprint - c.writers["/extractor/nodes.meta"] = &c.NodesMeta - c.writers["/extractor/nodes"] = &c.Nodes - c.writers["/extractor/barriers.meta"] = &c.BarriersMeta - c.writers["/extractor/barriers"] = &c.Barriers - c.writers["/extractor/traffic_lights.meta"] = &c.TrafficLightsMeta - c.writers["/extractor/traffic_lights"] = &c.TrafficLights - c.writers["/extractor/edges.meta"] = &c.EdgesMeta - c.writers["/extractor/edges"] = &c.Edges - c.writers["/extractor/annotations.meta"] = &c.AnnotationsMeta - c.writers["/extractor/annotations"] = &c.Annotations - - return &c -} - -func (c *Contents) validate() error { +// Validate checks whether the contents valid or not. +func (c *Contents) Validate() error { if !c.Fingerprint.IsValid() { return fmt.Errorf("invalid fingerprint %v", c.Fingerprint) } @@ -168,3 +128,14 @@ func (c *Contents) validate() error { return nil } + +// FindWriter find io.Writer for the specified name. +func (c *Contents) FindWriter(name string) (io.Writer, bool) { + w, b := c.writers[name] + return w, b +} + +// FilePath returns the file path that stores the contents. +func (c *Contents) FilePath() string { + return c.filePath +} diff --git a/integration/osrmfiles/dotosrmdottimestamp/contents.go b/integration/osrmfiles/dotosrmdottimestamp/contents.go new file mode 100644 index 00000000000..822d2a5ccd4 --- /dev/null +++ b/integration/osrmfiles/dotosrmdottimestamp/contents.go @@ -0,0 +1,70 @@ +package dotosrmdottimestamp + +import ( + "bytes" + "fmt" + "io" + + "github.com/Telenav/osrm-backend/integration/osrmfiles/fingerprint" + "github.com/Telenav/osrm-backend/integration/osrmfiles/meta" + "github.com/golang/glog" +) + +// Contents represents `.osrm.timestamp` file structure. +type Contents struct { + Fingerprint fingerprint.Fingerprint + TimestampMeta meta.Num + Timestamp bytes.Buffer + + // for internal implementation + writers map[string]io.Writer + filePath string +} + +// New creates an empty Contents for `.osrm.timestamp`. +func New(file string) *Contents { + c := Contents{} + + c.filePath = file + + // init writers + c.writers = map[string]io.Writer{} + c.writers["osrm_fingerprint.meta"] = &c.Fingerprint + c.writers["/common/timestamp.meta"] = &c.TimestampMeta + c.writers["/common/timestamp"] = &c.Timestamp + + return &c +} + +// Validate checks whether the contents valid or not. +func (c *Contents) Validate() error { + if !c.Fingerprint.IsValid() { + return fmt.Errorf("invalid fingerprint %v", c.Fingerprint) + } + if uint64(c.TimestampMeta) != uint64(c.Timestamp.Len()) { + return fmt.Errorf("timestamp meta not match, count in meta %d, but actual timestamp bytse count %d", c.TimestampMeta, c.Timestamp.Len()) + } + return nil +} + +// PrintSummary prints summary and head lines of contents. +func (c *Contents) PrintSummary(head int) { + glog.Infof("Loaded from %s\n", c.filePath) + glog.Infof(" %s\n", &c.Fingerprint) + + glog.Infof(" timestamp(a.k.a. data_version) meta %d count %d\n", c.TimestampMeta, c.Timestamp.Len()) + if head > 0 { + glog.Infof(" timestamp(a.k.a. data_version) %v\n", c.Timestamp.String()) + } +} + +// FindWriter find io.Writer for the specified name. +func (c *Contents) FindWriter(name string) (io.Writer, bool) { + w, b := c.writers[name] + return w, b +} + +// FilePath returns the file path that stores the contents. +func (c *Contents) FilePath() string { + return c.filePath +} diff --git a/integration/osrmfiles/tar_loader.go b/integration/osrmfiles/tar_loader.go new file mode 100644 index 00000000000..0b4c8c38006 --- /dev/null +++ b/integration/osrmfiles/tar_loader.go @@ -0,0 +1,49 @@ +package osrmfiles + +import ( + "archive/tar" + "io" + "os" + + "github.com/golang/glog" +) + +// Load `.osrm[.xxx]` tar file contents into its Contents structure. +func Load(contents ContentsOperator) error { + f, err := os.Open(contents.FilePath()) + defer f.Close() + if err != nil { + return err + } + glog.V(2).Infof("open %s succeed.\n", contents.FilePath()) + + // Open and iterate through the files in the archive. + tr := tar.NewReader(f) + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + glog.Fatal(err) + } + glog.V(1).Infof("%s\n", hdr.Name) + //writer, found := contents.writers[hdr.Name] + writer, found := contents.FindWriter(hdr.Name) + if !found { + glog.Warningf("unrecognized content in tar: %s", hdr.Name) + continue + } + + if _, err := io.Copy(writer, tr); err != nil { + glog.Fatal(err) + } + } + + // validate loaded contents + if err := contents.Validate(); err != nil { + return err + } + + return nil +}