From 8061068b7a691c6e9f8d570b9ce221fa2a8b7a36 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 16:50:50 -0700 Subject: [PATCH 1/8] feat: implement spartial index query based on google::s2 issue: https://github.com/Telenav/osrm-backend/issues/239 --- .../chargestations-connectivity-gen/doc.go | 17 ++ integration/go.mod | 1 + integration/go.sum | 3 + integration/service/poiloader/loader.go | 26 ++ integration/service/poiloader/poi_format.go | 64 +++++ .../service/poiloader/sample_input.json | 64 +++++ integration/service/spatialindexer/doc.go | 19 ++ .../service/spatialindexer/iterface.go | 30 ++ .../spatialindexer/s2indexer/builder.go | 48 ++++ .../spatialindexer/s2indexer/builder_test.go | 260 ++++++++++++++++++ .../spatialindexer/s2indexer/dumper.go | 146 ++++++++++ .../spatialindexer/s2indexer/dumper_test.go | 65 +++++ .../spatialindexer/s2indexer/indexer.go | 76 +++++ .../spatialindexer/s2indexer/spatial_query.go | 76 +++++ .../s2indexer/spatial_query_test.go | 245 +++++++++++++++++ 15 files changed, 1140 insertions(+) create mode 100644 integration/cmd/chargestations-connectivity-gen/doc.go create mode 100644 integration/service/poiloader/loader.go create mode 100644 integration/service/poiloader/poi_format.go create mode 100644 integration/service/poiloader/sample_input.json create mode 100644 integration/service/spatialindexer/doc.go create mode 100644 integration/service/spatialindexer/iterface.go create mode 100644 integration/service/spatialindexer/s2indexer/builder.go create mode 100644 integration/service/spatialindexer/s2indexer/builder_test.go create mode 100644 integration/service/spatialindexer/s2indexer/dumper.go create mode 100644 integration/service/spatialindexer/s2indexer/dumper_test.go create mode 100644 integration/service/spatialindexer/s2indexer/indexer.go create mode 100644 integration/service/spatialindexer/s2indexer/spatial_query.go create mode 100644 integration/service/spatialindexer/s2indexer/spatial_query_test.go diff --git a/integration/cmd/chargestations-connectivity-gen/doc.go b/integration/cmd/chargestations-connectivity-gen/doc.go new file mode 100644 index 00000000000..8e0f8b31e38 --- /dev/null +++ b/integration/cmd/chargestations-connectivity-gen/doc.go @@ -0,0 +1,17 @@ +// package main contains the tool of chargestation-connectivity generator +// stage 1: +// inputs is json file +// => convert to slice of [id:string,location: lat,lon] +// => calculate cellids for each point(for all levels) +// => build revese index for cellid -> ids +// stage 2: +// => iterate each point +// => generate a circle(s2::cap), find all cellids intersect with that circle +// => retrieve all ids +// => generate result of id(from), ids(all ids in certain distance) +// stage 3: +// => load data from file +// => for each line, its formid and all other ids +// => calculate distance between fromid and all other ids +// => sort result based on distance and write back to file +package main diff --git a/integration/go.mod b/integration/go.mod index bc2c88727a9..415bc0084aa 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -3,6 +3,7 @@ module github.com/Telenav/osrm-backend/integration go 1.13 require ( + github.com/golang/geo v0.0.0-20200319012246-673a6f80352d github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.1 diff --git a/integration/go.sum b/integration/go.sum index 8bb21f4ae9f..6c3a3f88da6 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1,6 +1,9 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Telenav/osrm-backend v5.22.0+incompatible h1:v5K4VmdxyQjFXN20ZGmk9MchGTHWWIMUXiXmAndNvlQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/golang/geo v0.0.0-20200319012246-673a6f80352d h1:C/hKUcHT483btRbeGkrRjJz+Zbcj8audldIi9tRJDCc= +github.com/golang/geo v0.0.0-20200319012246-673a6f80352d/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= diff --git a/integration/service/poiloader/loader.go b/integration/service/poiloader/loader.go new file mode 100644 index 00000000000..5f86e98bfa1 --- /dev/null +++ b/integration/service/poiloader/loader.go @@ -0,0 +1,26 @@ +package poiloader + +import ( + "encoding/json" + "io/ioutil" + + "github.com/golang/glog" +) + +// LoadPOI accepts json file recorded with poi data and returns deserialized result +func LoadData(filePath string) ([]Element, error) { + var elements []Element + + file, err := ioutil.ReadFile(filePath) + if err != nil { + glog.Errorf("While load file %s, met error %v\n", filePath, err) + return elements, err + } + + err = json.Unmarshal(file, &elements) + if err != nil { + glog.Errorf("While unmarshal json file %s, met error %v\n", filePath, err) + return elements, err + } + return elements, nil +} diff --git a/integration/service/poiloader/poi_format.go b/integration/service/poiloader/poi_format.go new file mode 100644 index 00000000000..95c3b5e4055 --- /dev/null +++ b/integration/service/poiloader/poi_format.go @@ -0,0 +1,64 @@ +package poiloader + +type Element struct { + ID int64 `json:"id"` + VendorCode VendorCode `json:"vendor_code"` + VendorPoiID string `json:"vendor_poi_id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + NavLat float64 `json:"nav_lat"` + NavLon float64 `json:"nav_lon"` + DisLat float64 `json:"dis_lat"` + DisLon float64 `json:"dis_lon"` + MapLinkID int64 `json:"map_link_id"` + SideOfStreet string `json:"side_of_street"` + Country Country `json:"country"` + SpaceID SpaceID `json:"space_id"` + AirportCode string `json:"airport_code"` + IsNational bool `json:"is_national"` + IsStateImportance bool `json:"is_state_importance"` + IsCityImportance bool `json:"is_city_importance"` + Fax string `json:"fax"` + Phone *CategoryIDGather `json:"phone"` + EncodedPhone string `json:"encoded_phone"` + Email string `json:"email"` + WebURL string `json:"web_url"` + CategoryIDGather *CategoryIDGather `json:"category_id_gather"` + ChainGather string `json:"chain_gather"` + RawCategoryGather string `json:"raw_category_gather"` + ChildGather string `json:"child_gather"` + ParentGather string `json:"parent_gather"` + Hilbert float64 `json:"hilbert"` + Amenity Amenity `json:"amenity"` +} + +type Amenity string + +const ( + ChargingStation Amenity = "charging_station" +) + +type Country string + +const ( + Usa Country = "USA" +) + +type SpaceID string + +const ( + UsaCA SpaceID = "USA_CA" +) + +type VendorCode string + +const ( + Noel VendorCode = "NOEL" + Noft VendorCode = "NOFT" + Nolp VendorCode = "NOLP" +) + +type CategoryIDGather struct { + Integer *int64 + String *string +} diff --git a/integration/service/poiloader/sample_input.json b/integration/service/poiloader/sample_input.json new file mode 100644 index 00000000000..6f83a702a3b --- /dev/null +++ b/integration/service/poiloader/sample_input.json @@ -0,0 +1,64 @@ +[ + { + "id":12345, + "vendor_code":"NOFT", + "vendor_poi_id":"NOFT-10_NREL99930@-@NOCO-37145190@-@NOCO-996595530", + "lat":0.48651, + "lon":0.66526, + "nav_lat":0.48651, + "nav_lon":0.66526, + "dis_lat":0.48651, + "dis_lon":0.66526, + "map_link_id":12345, + "side_of_street":"+", + "country":"USA", + "space_id":"USA_CA", + "airport_code":"NULL", + "is_national":false, + "is_state_importance":false, + "is_city_importance":false, + "fax":"NULL", + "phone":123456789, + "encoded_phone":"123456789", + "email":"NULL", + "web_url":"NULL", + "category_id_gather":"771``794``915", + "chain_gather":"1364@-@@-@", + "raw_category_gather":"NT@-@3578``NT@-@6000``NAICS@-@522110``NT@-@5540", + "child_gather":"NULL", + "parent_gather":"1718449@-@1``2216423@-@1", + "hilbert":2.32E+11, + "amenity":"charging_station" + }, + { + "id":12346, + "vendor_code":"NOFT", + "vendor_poi_id":"NOFT-10_NREL99930@-@NOCO-37145190@-@NOCO-996595530", + "lat":0.48651, + "lon":0.66526, + "nav_lat":0.48651, + "nav_lon":0.66526, + "dis_lat":0.48651, + "dis_lon":0.66526, + "map_link_id":12345, + "side_of_street":"+", + "country":"USA", + "space_id":"USA_CA", + "airport_code":"NULL", + "is_national":false, + "is_state_importance":false, + "is_city_importance":false, + "fax":"NULL", + "phone":123456789, + "encoded_phone":"123456789", + "email":"NULL", + "web_url":"NULL", + "category_id_gather":"771``794``915", + "chain_gather":"1364@-@@-@", + "raw_category_gather":"NT@-@3578``NT@-@6000``NAICS@-@522110``NT@-@5540", + "child_gather":"NULL", + "parent_gather":"1718449@-@1``2216423@-@1", + "hilbert":2.32E+11, + "amenity":"charging_station" + } +] \ No newline at end of file diff --git a/integration/service/spatialindexer/doc.go b/integration/service/spatialindexer/doc.go new file mode 100644 index 00000000000..2e98a65aaeb --- /dev/null +++ b/integration/service/spatialindexer/doc.go @@ -0,0 +1,19 @@ +// package spatialindexer answers query of nearest pois for given point +// +// Sample 1: Build connectivity for charge stations during pro-processing +// indexer := NewS2Indexer().Build(poiCsvFile) +// for _, stationPoint := range chargeStations { +// nearbyStations := indexer.FindNearByIDs(stationPoint, 800km, -1) +// rankedStations := indexer.RankingIDsByShortestDistance(stationPoint, nearbyStations) +// } +// +// +// Sample 2: Dump s2Indexer's content to folder +// indexer.Dump(folderPath) +// +// +// Sample 3: Query reachable charge stations with current energy level +// indexer := NewS2Indexer().Load(folderPath) +// nearbyStations := indexer.FindNearByIDs(currentPoint, currentEnergyLevel, -1) +// +package spatialindexer diff --git a/integration/service/spatialindexer/iterface.go b/integration/service/spatialindexer/iterface.go new file mode 100644 index 00000000000..e8213635271 --- /dev/null +++ b/integration/service/spatialindexer/iterface.go @@ -0,0 +1,30 @@ +package spatialindexer + +// Location for poi point +// @todo: will be replaced by the one in map +type Location struct { + Latitude float64 + Longitude float64 +} + +type PointInfo struct { + ID PointID + Location Location +} + +type RankedPointInfo struct { + PointInfo + Distance float64 +} + +type PointID int64 + +type Finder interface { + FindNearByIDs(center Location, radius float64, limitCount int) []PointInfo +} + +type Ranker interface { + RankingIDsByGreatCircleDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo + + RankingIDsByShortestDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo +} diff --git a/integration/service/spatialindexer/s2indexer/builder.go b/integration/service/spatialindexer/s2indexer/builder.go new file mode 100644 index 00000000000..c7ef012d7bf --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/builder.go @@ -0,0 +1,48 @@ +package s2indexer + +import ( + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" +) + +// https://s2geometry.io/resources/s2cell_statistics.html +// Level = 6 means average area size is 20754.64km2 +// Level = 9 means average area size is 324.29km2 +const minS2Level = 9 + +// Level = 19 means average area size is 309.27km2 +// Level = 20 means average area size is 77.32km2 +const maxS2Level = 20 + +func build(pois []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.CellID][]spatialindexer.PointID { + pointID2CellIDs := make(map[spatialindexer.PointID][]s2.CellID) + cellID2PointIDs := make(map[s2.CellID][]spatialindexer.PointID) + + for _, p := range pois { + leafCellID := s2.CellFromLatLng(s2.LatLngFromDegrees(p.Location.Latitude, p.Location.Longitude)).ID() + + var cellIDs []s2.CellID + // For level = 30, its parent equal to current + // So no need append leafCellID outside of for loop if maxLevel = 30 + for i := leafCellID.Level(); i > minLevel; i-- { + if i > maxLevel { + continue + } + parentCellID := leafCellID.Parent(i) + cellIDs = append(cellIDs, parentCellID) + } + pointID2CellIDs[p.ID] = cellIDs + } + + for pointID, cellIDs := range pointID2CellIDs { + for _, cellID := range cellIDs { + if _, ok := cellID2PointIDs[cellID]; !ok { + var pointIDs []spatialindexer.PointID + cellID2PointIDs[cellID] = pointIDs + } + cellID2PointIDs[cellID] = append(cellID2PointIDs[cellID], pointID) + } + } + + return cellID2PointIDs +} diff --git a/integration/service/spatialindexer/s2indexer/builder_test.go b/integration/service/spatialindexer/s2indexer/builder_test.go new file mode 100644 index 00000000000..4fa3c8e3847 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/builder_test.go @@ -0,0 +1,260 @@ +package s2indexer + +import ( + "reflect" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" +) + +// online tool for google s2: http://s2map.com/#order=latlng&mode=polygon&s2=false&points= +func TestBuild(t *testing.T) { + cases := []struct { + points []spatialindexer.PointInfo + expect map[s2.CellID][]spatialindexer.PointID + }{ + { + []spatialindexer.PointInfo{ + spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Latitude: 37.402701, + Longitude: -121.974096, + }, + }, + }, + map[s2.CellID][]spatialindexer.PointID{ + 9263834064756932608: {1}, // 4/0010133 + 9263851656942977024: {1}, // 4/00101332 + 9263847258896465920: {1}, // 4/001013321 + 9263843960361582592: {1}, // 4/0010133210 + 9263844784995303424: {1}, // 4/00101332103 + 9263844853714780160: {1}, // 4/001013321032 + 9263844836534910976: {1}, // 4/0010133210321 + 9263844823650009088: {1}, // 4/00101332103210 + 9263844822576267264: {1}, // 4/001013321032101 + 9263844823381573632: {1}, // 4/0010133210321013 + 9263844823180247040: {1}, // 4/00101332103210130 + 9263844823129915392: {1}, // 4/001013321032101300 + 9263844823134109696: {1}, // 4/0010133210321013002 + 9263844823130963968: {1}, // 4/00101332103210130020 + 9263844823130701824: {1}, // 4/001013321032101300201 + 9263844823130898432: {1}, // 4/0010133210321013002013 + 9263844823130947584: {1}, // 4/00101332103210130020133 + 9263844823130943488: {1}, // 4/001013321032101300201331 + 9263844823130944512: {1}, // 4/0010133210321013002013312 + 9263844823130943744: {1}, // 4/00101332103210130020133120 + 9263844823130943680: {1}, // 4/001013321032101300201331201 + 9263844823130943696: {1}, // 4/0010133210321013002013312012 + 9263844823130943708: {1}, // 4/00101332103210130020133120123 + 9263844823130943709: {1}, // 4/001013321032101300201331201232 + }, + }, + + // Test case of 2 points, both near to 4655 Great America Pkwy + { + []spatialindexer.PointInfo{ + spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Latitude: 37.402701, + Longitude: -121.974096, + }, + }, + spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Latitude: 37.403530, + Longitude: -121.969768, + }, + }, + }, + map[s2.CellID][]spatialindexer.PointID{ + 9263834064756932608: {1, 2}, // 4/0010133 + 9263851656942977024: {1, 2}, // 4/00101332 + 9263847258896465920: {1, 2}, // 4/001013321 + 9263843960361582592: {1, 2}, // 4/0010133210 + 9263844784995303424: {1, 2}, // 4/00101332103 + 9263844716275826688: {2}, // 4/001013321031 + 9263844733455695872: {2}, // 4/0010133210312 + 9263844746340597760: {2}, // 4/00101332103123 + 9263844749561823232: {2}, // 4/001013321031233 + 9263844749830258688: {2}, // 4/0010133210312332 + 9263844750031585280: {2}, // 4/00101332103123323 + 9263844749981253632: {2}, // 4/001013321031233230 + 9263844749993836544: {2}, // 4/0010133210312332303 + 9263844749996982272: {2}, // 4/00101332103123323033 + 9263844749996720128: {2}, // 4/001013321031233230331 + 9263844749996916736: {2}, // 4/0010133210312332303313 + 9263844749996900352: {2}, // 4/00101332103123323033131 + 9263844749996896256: {2}, // 4/001013321031233230331311 + 9263844749996893184: {2}, // 4/0010133210312332303313110 + 9263844749996892416: {2}, // 4/00101332103123323033131100 + 9263844749996892608: {2}, // 4/001013321031233230331311003 + 9263844749996892592: {2}, // 4/0010133210312332303313110031 + 9263844749996892588: {2}, // 4/00101332103123323033131100311 + 9263844749996892591: {2}, // 4/001013321031233230331311003113 + 9263844853714780160: {1}, // 4/001013321032 + 9263844836534910976: {1}, // 4/0010133210321 + 9263844823650009088: {1}, // 4/00101332103210 + 9263844822576267264: {1}, // 4/001013321032101 + 9263844823381573632: {1}, // 4/0010133210321013 + 9263844823180247040: {1}, // 4/00101332103210130 + 9263844823129915392: {1}, // 4/001013321032101300 + 9263844823134109696: {1}, // 4/0010133210321013002 + 9263844823130963968: {1}, // 4/00101332103210130020 + 9263844823130701824: {1}, // 4/001013321032101300201 + 9263844823130898432: {1}, // 4/0010133210321013002013 + 9263844823130947584: {1}, // 4/00101332103210130020133 + 9263844823130943488: {1}, // 4/001013321032101300201331 + 9263844823130944512: {1}, // 4/0010133210321013002013312 + 9263844823130943744: {1}, // 4/00101332103210130020133120 + 9263844823130943680: {1}, // 4/001013321032101300201331201 + 9263844823130943696: {1}, // 4/0010133210321013002013312012 + 9263844823130943708: {1}, // 4/00101332103210130020133120123 + 9263844823130943709: {1}, // 4/001013321032101300201331201232 + }, + }, + + // Test case of 4 points, points distrubted in CA, US + // Each points have about 100km distance with each other + { + []spatialindexer.PointInfo{ + spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Latitude: 37.651275, + Longitude: -122.413744, + }, + }, + spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Latitude: 36.776215, + Longitude: -121.733663, + }, + }, + spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Latitude: 36.122438, + Longitude: -121.022936, + }, + }, + spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Latitude: 35.365543, + Longitude: -120.850000, + }, + }, + }, + map[s2.CellID][]spatialindexer.PointID{ + 9263411852291866624: {2}, // 4/0010130 + 9263359075733733376: {2}, // 4/00101300 + 9263345881594200064: {2}, // 4/001013000 + 9263349180129083392: {2}, // 4/0010130003 + 9263349455006990336: {2}, // 4/00101300032 + 9263349386287513600: {2}, // 4/001013000321 + 9263349437827121152: {2}, // 4/0010130003213 + 9263349433532153856: {2}, // 4/00101300032131 + 9263349432458412032: {2}, // 4/001013000321311 + 9263349432726847488: {2}, // 4/0010130003213112 + 9263349432928174080: {2}, // 4/00101300032131123 + 9263349432877842432: {2}, // 4/001013000321311230 + 9263349432873648128: {2}, // 4/0010130003213112301 + 9263349432874696704: {2}, // 4/00101300032131123012 + 9263349432874434560: {2}, // 4/001013000321311230121 + 9263349432874500096: {2}, // 4/0010130003213112301212 + 9263349432874516480: {2}, // 4/00101300032131123012122 + 9263349432874528768: {2}, // 4/001013000321311230121223 + 9263349432874525696: {2}, // 4/0010130003213112301212230 + 9263349432874526464: {2}, // 4/00101300032131123012122303 + 9263349432874526528: {2}, // 4/001013000321311230121223032 + 9263349432874526544: {2}, // 4/0010130003213112301212230322 + 9263349432874526556: {2}, // 4/00101300032131123012122303223 + 9263349432874526559: {2}, // 4/001013000321311230121223032233 + 9263693327268577280: {1}, // 4/0010132 + 9263746103826710528: {1}, // 4/00101323 + 9263759297966243840: {1}, // 4/001013233 + 9263755999431360512: {1}, // 4/0010132330 + 9263756824065081344: {1}, // 4/00101323303 + 9263756755345604608: {1}, // 4/001013233031 + 9263756703805997056: {1}, // 4/0010132330310 + 9263756708100964352: {1}, // 4/00101323303102 + 9263756709174706176: {1}, // 4/001013233031022 + 9263756709980012544: {1}, // 4/0010132330310223 + 9263756709912903680: {1}, // 4/00101323303102231 + 9263756709862572032: {1}, // 4/001013233031022310 + 9263756709866766336: {1}, // 4/0010132330310223102 + 9263756709867814912: {1}, // 4/00101323303102231022 + 9263756709867028480: {1}, // 4/001013233031022310220 + 9263756709866962944: {1}, // 4/0010132330310223102201 + 9263756709866913792: {1}, // 4/00101323303102231022010 + 9263756709866909696: {1}, // 4/001013233031022310220101 + 9263756709866908672: {1}, // 4/0010132330310223102201011 + 9263756709866907904: {1}, // 4/00101323303102231022010110 + 9263756709866907840: {1}, // 4/001013233031022310220101101 + 9263756709866907856: {1}, // 4/0010132330310223102201011012 + 9263756709866907844: {1}, // 4/00101323303102231022010110120 + 9263756709866907845: {1}, // 4/001013233031022310220101101202 + 9264678489687064576: {3}, // 4/0010211 + 9264731266245197824: {3}, // 4/00102113 + 9264718072105664512: {3}, // 4/001021130 + 9264719171617292288: {3}, // 4/0010211302 + 9264718346983571456: {3}, // 4/00102113020 + 9264718415703048192: {3}, // 4/001021130202 + 9264718398523179008: {3}, // 4/0010211302021 + 9264718411408080896: {3}, // 4/00102113020213 + 9264718414629306368: {3}, // 4/001021130202133 + 9264718413824000000: {3}, // 4/0010211302021330 + 9264718414025326592: {3}, // 4/00102113020213303 + 9264718414042103808: {3}, // 4/001021130202133032 + 9264718414046298112: {3}, // 4/0010211302021330322 + 9264718414043152384: {3}, // 4/00102113020213303220 + 9264718414042365952: {3}, // 4/001021130202133032200 + 9264718414042169344: {3}, // 4/0010211302021330322000 + 9264718414042152960: {3}, // 4/00102113020213303220001 + 9264718414042140672: {3}, // 4/001021130202133032200010 + 9264718414042143744: {3}, // 4/0010211302021330322000103 + 9264718414042143488: {3}, // 4/00102113020213303220001031 + 9264718414042143296: {3}, // 4/001021130202133032200010310 + 9264718414042143280: {3}, // 4/0010211302021330322000103101 + 9264718414042143284: {3}, // 4/00102113020213303220001031012 + 9264718414042143285: {3}, // 4/001021130202133032200010310122 + 9290011237591023616: {4}, // 4/0013121 + 9290064014149156864: {4}, // 4/00131213 + 9290050820009623552: {4}, // 4/001312130 + 9290047521474740224: {4}, // 4/0013121300 + 9290048346108461056: {4}, // 4/00131213003 + 9290048139950030848: {4}, // 4/001312130030 + 9290048191489638400: {4}, // 4/0013121300303 + 9290048195784605696: {4}, // 4/00131213003032 + 9290048196858347520: {4}, // 4/001312130030322 + 9290048197663653888: {4}, // 4/0013121300303223 + 9290048197864980480: {4}, // 4/00131213003032233 + 9290048197881757696: {4}, // 4/001312130030322332 + 9290048197885952000: {4}, // 4/0013121300303223322 + 9290048197882806272: {4}, // 4/00131213003032233220 + 9290048197883068416: {4}, // 4/001312130030322332202 + 9290048197883002880: {4}, // 4/0013121300303223322021 + 9290048197883052032: {4}, // 4/00131213003032233220213 + 9290048197883039744: {4}, // 4/001312130030322332202130 + 9290048197883040768: {4}, // 4/0013121300303223322021302 + 9290048197883041024: {4}, // 4/00131213003032233220213022 + 9290048197883041216: {4}, // 4/001312130030322332202130223 + 9290048197883041200: {4}, // 4/0013121300303223322021302231 + 9290048197883041212: {4}, // 4/00131213003032233220213022313 + 9290048197883041211: {4}, // 4/001312130030322332202130223131 + }, + }, + } + + for i, c := range cases { + actual := build(c.points, 6, 30) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("parse case %d \n%v, expect\n %v \n but got %v", i, c, c.expect, actual) + } + } +} diff --git a/integration/service/spatialindexer/s2indexer/dumper.go b/integration/service/spatialindexer/s2indexer/dumper.go new file mode 100644 index 00000000000..8600e0d74c6 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/dumper.go @@ -0,0 +1,146 @@ +package s2indexer + +import ( + "bytes" + "encoding/gob" + "io/ioutil" + "os" + "strings" + + "github.com/Telenav/osrm-backend/integration/pkg/api" + "github.com/golang/glog" +) + +const cellID2PointIDsFileName = "cellID2PointIDs.gob" +const pointID2LocationFileName = "pointID2Location.gob" + +func serializeS2Indexer(indexer *s2Indexer, folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath = folderPath + "/" + } + + if err := serializeCellID2PointIDs(indexer, folderPath); err != nil { + return err + } + + if err := serializePointID2Location(indexer, folderPath); err != nil { + return err + } + + glog.Infof("Successfully serialize s2Indexer to folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", + folderPath, len(indexer.cellID2PointIDs), len(indexer.pointID2Location)) + + return nil +} + +func serializeCellID2PointIDs(indexer *s2Indexer, folderPath string) error { + buf := new(bytes.Buffer) + encoder := gob.NewEncoder(buf) + err := encoder.Encode(indexer.cellID2PointIDs) + if err != nil { + glog.Errorf("During encode s2Indexer's cellID2PointIDs met error %v", err) + return err + } + if err = ioutil.WriteFile(folderPath+cellID2PointIDsFileName, buf.Bytes(), 0644); err != nil { + glog.Errorf("During dump s2Indexer's cellID2PointIDs met error %v", err) + return err + } + return nil +} + +func serializePointID2Location(indexer *s2Indexer, folderPath string) error { + buf := new(bytes.Buffer) + encoder := gob.NewEncoder(buf) + err := encoder.Encode(indexer.pointID2Location) + if err != nil { + glog.Errorf("During encode s2Indexer's pointID2Location met error %v", err) + return err + } + if err = ioutil.WriteFile(folderPath+pointID2LocationFileName, buf.Bytes(), 0644); err != nil { + glog.Errorf("During dump s2Indexer's pointID2Location met error %v", err) + return err + } + return nil +} + +func deSerializeS2Indexer(indexer *s2Indexer, folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath = folderPath + "/" + } + + if err := deSerializeCellID2PointIDs(indexer, folderPath); err != nil { + return err + } + + if err := deSerializePointID2Location(indexer, folderPath); err != nil { + return err + } + + glog.Infof("Successfully deserialize s2Indexer from folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", + folderPath, len(indexer.cellID2PointIDs), len(indexer.pointID2Location)) + + return nil +} + +func deSerializeCellID2PointIDs(indexer *s2Indexer, folderPath string) error { + byteArray, err := ioutil.ReadFile(folderPath + cellID2PointIDsFileName) + if err != nil { + glog.Errorf("During load s2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) + return err + } + buf := bytes.NewBuffer(byteArray) + decoder := gob.NewDecoder(buf) + err = decoder.Decode(&indexer.cellID2PointIDs) + if err != nil { + glog.Errorf("During decode s2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) + return err + } + + return nil +} + +func deSerializePointID2Location(indexer *s2Indexer, folderPath string) error { + byteArray, err := ioutil.ReadFile(folderPath + pointID2LocationFileName) + if err != nil { + glog.Errorf("During load s2Indexer's pointID2Location from %s met error %v", folderPath, err) + return err + } + buf := bytes.NewBuffer(byteArray) + decoder := gob.NewDecoder(buf) + err = decoder.Decode(&indexer.pointID2Location) + if err != nil { + glog.Errorf("During decode s2Indexer's pointID2Location from %s met error %v", folderPath, err) + return err + } + return nil + +} + +func removeAllDumpFiles(folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath = folderPath + "/" + } + + _, err := os.Stat(folderPath + cellID2PointIDsFileName) + if !os.IsNotExist(err) { + err = os.Remove(folderPath + cellID2PointIDsFileName) + if err != nil { + glog.Errorf("Remove file failed %s\n", folderPath+cellID2PointIDsFileName) + return err + } + } else { + glog.Warningf("There is no %s file in folder %s\n", cellID2PointIDsFileName, folderPath) + } + + _, err = os.Stat(folderPath + pointID2LocationFileName) + if !os.IsNotExist(err) { + err = os.Remove(folderPath + pointID2LocationFileName) + if err != nil { + glog.Errorf("Remove file failed %s\n", folderPath+pointID2LocationFileName) + return err + } + } else { + glog.Warningf("There is no %s file in folder %s\n", pointID2LocationFileName, folderPath) + } + return nil +} diff --git a/integration/service/spatialindexer/s2indexer/dumper_test.go b/integration/service/spatialindexer/s2indexer/dumper_test.go new file mode 100644 index 00000000000..a8393dfed05 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/dumper_test.go @@ -0,0 +1,65 @@ +package s2indexer + +import ( + "os" + "reflect" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" +) + +func TestDumper(t *testing.T) { + cases := []s2Indexer{ + s2Indexer{ + cellID2PointIDs: map[s2.CellID][]spatialindexer.PointID{ + 9263834064756932608: {1, 2, 3}, // 4/0010133 + 9263851656942977024: {1}, // 4/00101332 + 9263847258896465920: {2}, // 4/001013321 + 9263843960361582592: {3}, // 4/0010133210 + }, + pointID2Location: map[spatialindexer.PointID]spatialindexer.Location{ + 1: spatialindexer.Location{ + Latitude: 11.11, + Longitude: 11.11, + }, + 2: spatialindexer.Location{ + Latitude: 22.22, + Longitude: 22.22, + }, + 3: spatialindexer.Location{ + Latitude: 33.33, + Longitude: 33.33, + }, + }, + }, + } + // check whether curent folder is writeable + path, _ := os.Getwd() + _, err := os.OpenFile(path+"/tmp", os.O_RDONLY|os.O_CREATE, 0644) + if err != nil { + return + } + err = os.Remove(path + "/tmp") + + removeAllDumpFiles(path) + for _, c := range cases { + err := serializeS2Indexer(&c, path) + if err != nil { + t.Errorf("During running serializeS2Indexer for case %v, met error %v", c, err) + } + + actual := NewS2Indexer() + err = deSerializeS2Indexer(actual, path) + if err != nil { + t.Errorf("During running deSerializeS2Indexer for case %v, met error %v", c, err) + } + + if !reflect.DeepEqual(actual, &c) { + t.Errorf("Expect result \n%v but got \n%v\n", &c, actual) + } + + removeAllDumpFiles(path) + } + +} diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go new file mode 100644 index 00000000000..2f524529d0d --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -0,0 +1,76 @@ +package s2indexer + +import ( + "github.com/Telenav/osrm-backend/integration/service/poiloader" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" + "github.com/golang/glog" +) + +type s2Indexer struct { + cellID2PointIDs map[s2.CellID][]spatialindexer.PointID + pointID2Location map[spatialindexer.PointID]spatialindexer.Location +} + +// NewS2Indexer generates spatial indexer based on google s2 +func NewS2Indexer() *s2Indexer { + return &s2Indexer{} +} + +// Build constructs S2 indexer +func (indexer *s2Indexer) Build(filePath string) *s2Indexer { + records, err := poiloader.LoadData(filePath) + if err != nil || len(records) == 0 { + return nil + } + + var pointInfos []spatialindexer.PointInfo + for _, record := range records { + pointInfo := spatialindexer.PointInfo{ + ID: elementID2PointID(record.ID), + Location: spatialindexer.Location{ + Latitude: record.Lat, + Longitude: record.Lon, + }, + } + pointInfos = append(pointInfos, pointInfo) + + indexer.pointID2Location[elementID2PointID(record.ID)] = spatialindexer.Location{ + Latitude: record.Lat, + Longitude: record.Lon, + } + } + + indexer.cellID2PointIDs = build(pointInfos, minS2Level, maxS2Level) + return indexer +} + +// Load s2Indexer's data from contents recorded in folder +func (indexer *s2Indexer) Load(folderPath string) *s2Indexer { + if err := deSerializeS2Indexer(indexer, folderPath); err != nil { + glog.Errorf("Load s2Indexer's data from folder %s failed, err=%v\n", folderPath, err) + return nil + } + return indexer +} + +// Dump s2Indexer's content into folderPath +func (indexer *s2Indexer) Dump(folderPath string) { + if err := serializeS2Indexer(indexer, folderPath); err != nil { + glog.Errorf("Dump s2Indexer's data to folder %s failed, err=%v\n", folderPath, err) + } +} + +func (indexer s2Indexer) getPointLocationByPointID(id spatialindexer.PointID) (spatialindexer.Location, bool) { + location, ok := indexer.pointID2Location[id] + return location, ok +} + +func (indexer s2Indexer) getPointIDsByS2CellID(cellid s2.CellID) ([]spatialindexer.PointID, bool) { + pointIDs, ok := indexer.cellID2PointIDs[cellid] + return pointIDs, ok +} + +func elementID2PointID(id int64) spatialindexer.PointID { + return (spatialindexer.PointID)(id) +} diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go new file mode 100644 index 00000000000..0024a4fd4f0 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -0,0 +1,76 @@ +package s2indexer + +import ( + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s1" + "github.com/golang/geo/s2" + "github.com/golang/glog" +) + +const maxCellCount = 200 + +const s2EarthRadiusInMeters = 6371010.0 + +func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) []s2.CellID { + regionCover := &s2.RegionCoverer{ + MinLevel: minS2Level, + MaxLevel: 20, + MaxCells: maxCellCount} + center := s2.PointFromLatLng(s2.LatLngFromDegrees(point.Latitude, point.Longitude)) + radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) + region := s2.Region(s2.CapFromCenterAngle(center, radius)) + cellUnion := regionCover.Covering(region) + generateDebugInfo4CellIDs(([]s2.CellID)(cellUnion)) + return ([]s2.CellID)(cellUnion) +} + +func queryNearByPoints(indexer *s2Indexer, point spatialindexer.Location, radius float64) []spatialindexer.PointInfo { + var result []spatialindexer.PointInfo + cellIDs := queryNearByS2Cells(point, radius) + for _, cellID := range cellIDs { + pointIDs, hasCellID := indexer.getPointIDsByS2CellID(cellID) + if !hasCellID { + continue + } + + for _, pointID := range pointIDs { + location, hasPointID := indexer.getPointLocationByPointID(pointID) + if !hasPointID { + glog.Errorf("In queryNearByPoints, use incorrect pointID %v to query s2Indexer\n", pointID) + continue + } + + result = append(result, spatialindexer.PointInfo{ + ID: pointID, + Location: location, + }) + } + } + + return result +} + +func generateDebugInfo4CellIDs(cellIDs []s2.CellID) { + glog.Info("=================================\n") + glog.Info("generateDebugInfo4CellIDs\n") + for _, cellID := range cellIDs { + glog.Infof("CellID value = %d(uint64), string = %v, token = %s, level = %d\n", + (uint64)(cellID), cellID, cellID.ToToken(), cellID.Level()) + } + glog.Info("=================================\n") +} + +// Generate sample url like: http://s2.sidewalklabs.com/regioncoverer/?cells=89c2584b54,89c2584d,89c25852c,89c259a5,89c259ac,89c259b4,89c259bc,89c259c7,89c259c9,89c259ca4 +func generateDebugURL(cellIDs []s2.CellID) string { + var url string + + if len(cellIDs) == 0 { + return url + } + + url += "http://s2.sidewalklabs.com/regioncoverer/?cells=" + for _, cellID := range cellIDs { + url += cellID.ToToken() + "," + } + return url +} diff --git a/integration/service/spatialindexer/s2indexer/spatial_query_test.go b/integration/service/spatialindexer/s2indexer/spatial_query_test.go new file mode 100644 index 00000000000..2b9599610a0 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/spatial_query_test.go @@ -0,0 +1,245 @@ +package s2indexer + +import ( + "reflect" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" + "github.com/golang/glog" +) + +func TestSpatialIndexQuery1(t *testing.T) { + fakeIndexer1 := s2Indexer{ + cellID2PointIDs: map[s2.CellID][]spatialindexer.PointID{ + 9263834064756932608: {1, 2}, // 4/0010133 + 9263851656942977024: {1, 2}, // 4/00101332 + 9263847258896465920: {1, 2}, // 4/001013321 + 9263843960361582592: {1, 2}, // 4/0010133210 + 9263844784995303424: {1, 2}, // 4/00101332103 + 9263844716275826688: {2}, // 4/001013321031 + 9263844733455695872: {2}, // 4/0010133210312 + 9263844746340597760: {2}, // 4/00101332103123 + 9263844749561823232: {2}, // 4/001013321031233 + 9263844749830258688: {2}, // 4/0010133210312332 + 9263844750031585280: {2}, // 4/00101332103123323 + 9263844749981253632: {2}, // 4/001013321031233230 + 9263844749993836544: {2}, // 4/0010133210312332303 + 9263844749996982272: {2}, // 4/00101332103123323033 + 9263844749996720128: {2}, // 4/001013321031233230331 + 9263844749996916736: {2}, // 4/0010133210312332303313 + 9263844749996900352: {2}, // 4/00101332103123323033131 + 9263844749996896256: {2}, // 4/001013321031233230331311 + 9263844749996893184: {2}, // 4/0010133210312332303313110 + 9263844749996892416: {2}, // 4/00101332103123323033131100 + 9263844749996892608: {2}, // 4/001013321031233230331311003 + 9263844749996892592: {2}, // 4/0010133210312332303313110031 + 9263844749996892588: {2}, // 4/00101332103123323033131100311 + 9263844749996892591: {2}, // 4/001013321031233230331311003113 + 9263844853714780160: {1}, // 4/001013321032 + 9263844836534910976: {1}, // 4/0010133210321 + 9263844823650009088: {1}, // 4/00101332103210 + 9263844822576267264: {1}, // 4/001013321032101 + 9263844823381573632: {1}, // 4/0010133210321013 + 9263844823180247040: {1}, // 4/00101332103210130 + 9263844823129915392: {1}, // 4/001013321032101300 + 9263844823134109696: {1}, // 4/0010133210321013002 + 9263844823130963968: {1}, // 4/00101332103210130020 + 9263844823130701824: {1}, // 4/001013321032101300201 + 9263844823130898432: {1}, // 4/0010133210321013002013 + 9263844823130947584: {1}, // 4/00101332103210130020133 + 9263844823130943488: {1}, // 4/001013321032101300201331 + 9263844823130944512: {1}, // 4/0010133210321013002013312 + 9263844823130943744: {1}, // 4/00101332103210130020133120 + 9263844823130943680: {1}, // 4/001013321032101300201331201 + 9263844823130943696: {1}, // 4/0010133210321013002013312012 + 9263844823130943708: {1}, // 4/00101332103210130020133120123 + 9263844823130943709: {1}, // 4/001013321032101300201331201232 + }, + pointID2Location: map[spatialindexer.PointID]spatialindexer.Location{ + 1: spatialindexer.Location{ + Latitude: 37.402701, + Longitude: -121.974096, + }, + 2: spatialindexer.Location{ + Latitude: 37.403530, + Longitude: -121.969768, + }, + }, + } + + // center in 4655 great america pkwy + center := spatialindexer.Location{ + Latitude: 37.402799, + Longitude: -121.969861, + } + + expect := []spatialindexer.PointInfo{ + spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Latitude: 37.402701, + Longitude: -121.974096, + }, + }, + spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Latitude: 37.40353, + Longitude: -121.969768, + }, + }, + } + + actual := queryNearByPoints(&fakeIndexer1, center, 10000) + + if !reflect.DeepEqual(actual, expect) { + t.Errorf("Expect result is \n%v but got \n%v\n", actual, expect) + } +} + +//More information could go to here: https://github.com/Telenav/osrm-backend/issues/236#issuecomment-603533484 +func TestQueryNearByS2Cells1(t *testing.T) { + // center in 4655 great america pkwy + center := spatialindexer.Location{ + Latitude: 37.402799, + Longitude: -121.969861, + } + actualCellIDs := queryNearByS2Cells(center, 1600) + glog.Infof("\nTest URL is %s\n", generateDebugURL(actualCellIDs)) + + expectCellIDs := []s2.CellID{ + 9263843046942834688, // token = 808fc82b54 + 9263843047026720768, // token = 808fc82b59 + 9263843047131578368, // token = 808fc82b5f4 + 9263843050566713344, // token = 808fc82c2c + 9263843050700931072, // token = 808fc82c34 + 9263843050835148800, // token = 808fc82c3c + 9263843051170693120, // token = 808fc82c5 + 9263843051506237440, // token = 808fc82c64 + 9263843051590123520, // token = 808fc82c69 + 9263843053049741312, // token = 808fc82cc + 9263843055197224960, // token = 808fc82d4 + 9263843057344708608, // token = 808fc82dc + 9263843058686885888, // token = 808fc82e1 + 9263843060364607488, // token = 808fc82e74 + 9263843060448493568, // token = 808fc82e79 + 9263843072377094144, // token = 808fc8314 + 9263843073719271424, // token = 808fc8319 + 9263843074256142336, // token = 808fc831b + 9263843074591686656, // token = 808fc831c4 + 9263843074994339840, // token = 808fc831dc + 9263843075329884160, // token = 808fc831f + 9263843079893286912, // token = 808fc833 + 9263843088483221504, // token = 808fc835 + 9263843093046624256, // token = 808fc8361 + 9263843093365391360, // token = 808fc83623 + 9263843093784821760, // token = 808fc8363c + 9263843093919039488, // token = 808fc83644 + 9263843094321692672, // token = 808fc8365c + 9263843094657236992, // token = 808fc8367 + 9263843095999414272, // token = 808fc836c + 9263843176525856768, // token = 808fc8497fc + 9263843176798486528, // token = 808fc8499 + 9263843178409099264, // token = 808fc849f + 9263843182972502016, // token = 808fc84b + 9263843188810973184, // token = 808fc84c5c + 9263843189146517504, // token = 808fc84c7 + 9263843190488694784, // token = 808fc84cc + 9263843192032198656, // token = 808fc84d1c + 9263843192367742976, // token = 808fc84d3 + 9263843192904613888, // token = 808fc84d5 + 9263844597090484224, // token = 808fc9944 + 9263844598432661504, // token = 808fc9949 + 9263844599036641280, // token = 808fc994b4 + 9263844599170859008, // token = 808fc994bc + 9263844599506403328, // token = 808fc994d + 9263844600043274240, // token = 808fc994f + 9263844601385451520, // token = 808fc9954 + 9263844603532935168, // token = 808fc995c + 9263844608901644288, // token = 808fc997 + 9263844614270353408, // token = 808fc9984 + 9263844615612530688, // token = 808fc9989 + 9263844616149401600, // token = 808fc998b + 9263844616484945920, // token = 808fc998c4 + 9263844617223143424, // token = 808fc998f + 9263844617558687744, // token = 808fc99904 + 9263844620444368896, // token = 808fc999b + 9263844620981239808, // token = 808fc999d + 9263844621316784128, // token = 808fc999e4 + 9263844621451001856, // token = 808fc999ec + 9263844653193494528, // token = 808fc9a15 + 9263844653730365440, // token = 808fc9a17 + 9263844654267236352, // token = 808fc9a19 + 9263844655944957952, // token = 808fc9a1f4 + 9263844656028844032, // token = 808fc9a1f9 + 9263844660441251840, // token = 808fc9a3 + 9263844669031186432, // token = 808fc9a5 + 9263844673594589184, // token = 808fc9a61 + 9263844675205201920, // token = 808fc9a67 + 9263844675742072832, // token = 808fc9a69 + 9263844676278943744, // token = 808fc9a6b + 9263844684801769472, // token = 808fc9a8ac + 9263844684935987200, // token = 808fc9a8b4 + 9263844685316620288, // token = 808fc9a8cab + 9263844699901263872, // token = 808fc9ac3 + 9263844703592251392, // token = 808fc9ad0c + 9263844704934428672, // token = 808fc9ad5c + 9263844705269972992, // token = 808fc9ad7 + 9263844706612150272, // token = 808fc9adc + 9263844708759633920, // token = 808fc9ae4 + 9263844710101811200, // token = 808fc9ae9 + 9263844710840008704, // token = 808fc9aebc + 9263844711175553024, // token = 808fc9aed + 9263844711712423936, // token = 808fc9aef + 9263844713054601216, // token = 808fc9af4 + 9263844715202084864, // token = 808fc9afc + 9263844733455695872, // token = 808fc9b4 + 9263844767815434240, // token = 808fc9bc + 9263844789290270720, // token = 808fc9c1 + 9263844794658979840, // token = 808fc9c24 + 9263844795934048256, // token = 808fc9c28c + 9263844796851552256, // token = 808fc9c2c2b + 9263844797276225536, // token = 808fc9c2dc + 9263844797611769856, // token = 808fc9c2f + 9263844798953947136, // token = 808fc9c34 + 9263844801101430784, // token = 808fc9c3c + 9263844806470139904, // token = 808fc9c5 + 9263844815060074496, // token = 808fc9c7 + 9263844836534910976, // token = 808fc9cc + 9263844858009747456, // token = 808fc9d1 + 9263844866599682048, // token = 808fc9d3 + 9263844871163084800, // token = 808fc9d41 + 9263844872773697536, // token = 808fc9d47 + 9263844873310568448, // token = 808fc9d49 + 9263844873646112768, // token = 808fc9d4a4 + 9263844873780330496, // token = 808fc9d4ac + 9263844882437373952, // token = 808fc9d6b + 9263844882907136000, // token = 808fc9d6cc + 9263844883041353728, // token = 808fc9d6d4 + 9263844884517748736, // token = 808fc9d72c + 9263844896865779712, // token = 808fc9da0c + 9263844896999997440, // token = 808fc9da14 + 9263844898006630400, // token = 808fc9da5 + 9263844898543501312, // token = 808fc9da7 + 9263844899885678592, // token = 808fc9dac + 9263844902033162240, // token = 808fc9db4 + 9263844903710883840, // token = 808fc9dba4 + 9263844903845101568, // token = 808fc9dbac + 9263844907921965056, // token = 808fc9dc9f + 9263844908207177728, // token = 808fc9dcb + 9263844908676939776, // token = 808fc9dccc + 9263844964779950080, // token = 808fc9e9dc + 9263844965115494400, // token = 808fc9e9f + 9263844966457671680, // token = 808fc9ea4 + 9263844968605155328, // token = 808fc9eac + 9263844969947332608, // token = 808fc9eb1 + 9263844970484203520, // token = 808fc9eb3 + 9263844970937188352, // token = 808fc9eb4b + 9263844972564578304, // token = 808fc9ebac + } + + if !reflect.DeepEqual(actualCellIDs, expectCellIDs) { + t.Errorf("Expect result is \n%v but got \n%v\n", expectCellIDs, actualCellIDs) + } +} From 8f9e8684ea8686f7535a400612d2b5b8dd113abc Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 16:57:18 -0700 Subject: [PATCH 2/8] fix: remove hard code value. issue: https://github.com/Telenav/osrm-backend/issues/239 --- integration/service/spatialindexer/s2indexer/spatial_query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go index 0024a4fd4f0..3c2d903cfd9 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -14,7 +14,7 @@ const s2EarthRadiusInMeters = 6371010.0 func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) []s2.CellID { regionCover := &s2.RegionCoverer{ MinLevel: minS2Level, - MaxLevel: 20, + MaxLevel: maxS2Level, MaxCells: maxCellCount} center := s2.PointFromLatLng(s2.LatLngFromDegrees(point.Latitude, point.Longitude)) radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) From 459b170aafec0b51d2f748faf72e1057ecef0a8d Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 18:01:24 -0700 Subject: [PATCH 3/8] fix: refactor code. --- integration/service/spatialindexer/doc.go | 9 +++++---- integration/service/spatialindexer/iterface.go | 17 ++++++++++++++--- .../service/spatialindexer/s2indexer/builder.go | 6 +++--- .../spatialindexer/s2indexer/builder_test.go | 2 +- .../service/spatialindexer/s2indexer/dumper.go | 1 + .../spatialindexer/s2indexer/spatial_query.go | 1 - 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/integration/service/spatialindexer/doc.go b/integration/service/spatialindexer/doc.go index 2e98a65aaeb..3e848cc3a20 100644 --- a/integration/service/spatialindexer/doc.go +++ b/integration/service/spatialindexer/doc.go @@ -1,6 +1,7 @@ -// package spatialindexer answers query of nearest pois for given point +// package spatialindexer answers query of nearest points(place, point of interest) for conditions +// such as center location, radius, etc // -// Sample 1: Build connectivity for charge stations during pro-processing +// Sample Scenario 1: Build connectivity for charge stations during pro-processing // indexer := NewS2Indexer().Build(poiCsvFile) // for _, stationPoint := range chargeStations { // nearbyStations := indexer.FindNearByIDs(stationPoint, 800km, -1) @@ -8,11 +9,11 @@ // } // // -// Sample 2: Dump s2Indexer's content to folder +// Sample Scenario 2: Dump s2Indexer's content to folder // indexer.Dump(folderPath) // // -// Sample 3: Query reachable charge stations with current energy level +// Sample Scenario 3: Query reachable charge stations with current energy level // indexer := NewS2Indexer().Load(folderPath) // nearbyStations := indexer.FindNearByIDs(currentPoint, currentEnergyLevel, -1) // diff --git a/integration/service/spatialindexer/iterface.go b/integration/service/spatialindexer/iterface.go index e8213635271..dfcca077f7c 100644 --- a/integration/service/spatialindexer/iterface.go +++ b/integration/service/spatialindexer/iterface.go @@ -7,24 +7,35 @@ type Location struct { Longitude float64 } +// PointInfo records point related information such as ID and location type PointInfo struct { ID PointID Location Location } +// RankedPointInfo used to record ranking result, distance to specific point could be used for ranking type RankedPointInfo struct { PointInfo Distance float64 } +// PointID defines ID for given point(location, point of interest) +// Only the data used for pre-processing contains valid PointID type PointID int64 +// Finder answers special query type Finder interface { - FindNearByIDs(center Location, radius float64, limitCount int) []PointInfo + + // FindNearByIDs returns a group of points near to given center location + FindNearByPointIDs(center Location, radius float64, limitCount int) []PointInfo } +// Ranker used to ranking a group of points type Ranker interface { - RankingIDsByGreatCircleDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo - RankingIDsByShortestDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo + // RankPointIDsByGreatCircleDistance ranks a group of points based on great circle distance to given location + RankPointIDsByGreatCircleDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo + + // RankPointIDsByShortestDistance ranks a group of points based on shortest path distance to given location + RankPointIDsByShortestDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo } diff --git a/integration/service/spatialindexer/s2indexer/builder.go b/integration/service/spatialindexer/s2indexer/builder.go index c7ef012d7bf..eed79895dbb 100644 --- a/integration/service/spatialindexer/s2indexer/builder.go +++ b/integration/service/spatialindexer/s2indexer/builder.go @@ -14,16 +14,16 @@ const minS2Level = 9 // Level = 20 means average area size is 77.32km2 const maxS2Level = 20 -func build(pois []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.CellID][]spatialindexer.PointID { +func build(points []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.CellID][]spatialindexer.PointID { pointID2CellIDs := make(map[spatialindexer.PointID][]s2.CellID) cellID2PointIDs := make(map[s2.CellID][]spatialindexer.PointID) - for _, p := range pois { + for _, p := range points { leafCellID := s2.CellFromLatLng(s2.LatLngFromDegrees(p.Location.Latitude, p.Location.Longitude)).ID() var cellIDs []s2.CellID // For level = 30, its parent equal to current - // So no need append leafCellID outside of for loop if maxLevel = 30 + // So no need append leafCellID into cellIDs outside of for loop for i := leafCellID.Level(); i > minLevel; i-- { if i > maxLevel { continue diff --git a/integration/service/spatialindexer/s2indexer/builder_test.go b/integration/service/spatialindexer/s2indexer/builder_test.go index 4fa3c8e3847..7a54a8862d9 100644 --- a/integration/service/spatialindexer/s2indexer/builder_test.go +++ b/integration/service/spatialindexer/s2indexer/builder_test.go @@ -8,7 +8,7 @@ import ( "github.com/golang/geo/s2" ) -// online tool for google s2: http://s2map.com/#order=latlng&mode=polygon&s2=false&points= +// online tool for google s2: http://s2.sidewalklabs.com/regioncoverer/ func TestBuild(t *testing.T) { cases := []struct { points []spatialindexer.PointInfo diff --git a/integration/service/spatialindexer/s2indexer/dumper.go b/integration/service/spatialindexer/s2indexer/dumper.go index 8600e0d74c6..d70fa978785 100644 --- a/integration/service/spatialindexer/s2indexer/dumper.go +++ b/integration/service/spatialindexer/s2indexer/dumper.go @@ -142,5 +142,6 @@ func removeAllDumpFiles(folderPath string) error { } else { glog.Warningf("There is no %s file in folder %s\n", pointID2LocationFileName, folderPath) } + return nil } diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go index 3c2d903cfd9..0cd26b65d75 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -20,7 +20,6 @@ func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) [ radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) region := s2.Region(s2.CapFromCenterAngle(center, radius)) cellUnion := regionCover.Covering(region) - generateDebugInfo4CellIDs(([]s2.CellID)(cellUnion)) return ([]s2.CellID)(cellUnion) } From cb08f933845b169010afea0e3087334a9a2e3fb5 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 18:28:33 -0700 Subject: [PATCH 4/8] fix: remove local changes. --- integration/go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/go.sum b/integration/go.sum index 6c3a3f88da6..4eaede4a194 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1,6 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Telenav/osrm-backend v5.22.0+incompatible h1:v5K4VmdxyQjFXN20ZGmk9MchGTHWWIMUXiXmAndNvlQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/golang/geo v0.0.0-20200319012246-673a6f80352d h1:C/hKUcHT483btRbeGkrRjJz+Zbcj8audldIi9tRJDCc= github.com/golang/geo v0.0.0-20200319012246-673a6f80352d/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= From 6de90a82c465629c00d318c2e8b8bd5261c3b52f Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 18:32:55 -0700 Subject: [PATCH 5/8] fix: update comments --- integration/service/poiloader/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/service/poiloader/loader.go b/integration/service/poiloader/loader.go index 5f86e98bfa1..7190c30a2fe 100644 --- a/integration/service/poiloader/loader.go +++ b/integration/service/poiloader/loader.go @@ -7,7 +7,7 @@ import ( "github.com/golang/glog" ) -// LoadPOI accepts json file recorded with poi data and returns deserialized result +// LoadData accepts json file with points data and returns deserialized result func LoadData(filePath string) ([]Element, error) { var elements []Element From 4dcf8f0c836b67c8001aec77de0cdae1fc5a980f Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 19:55:37 -0700 Subject: [PATCH 6/8] fix: add fake main to pass ci. --- integration/cmd/chargestations-connectivity-gen/main.go | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 integration/cmd/chargestations-connectivity-gen/main.go diff --git a/integration/cmd/chargestations-connectivity-gen/main.go b/integration/cmd/chargestations-connectivity-gen/main.go new file mode 100644 index 00000000000..934fe5d5111 --- /dev/null +++ b/integration/cmd/chargestations-connectivity-gen/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "fmt" +) + +func main() { + fmt.Print("Hello World!") +} From 240bb5b0a90080d6686191c2ff09c2d5a9160d3b Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 20:31:14 -0700 Subject: [PATCH 7/8] fix: fix issue in unit test. Build()'s result no need to guarantee the sequence --- .../service/spatialindexer/s2indexer/builder_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/integration/service/spatialindexer/s2indexer/builder_test.go b/integration/service/spatialindexer/s2indexer/builder_test.go index 7a54a8862d9..92b5c1a7502 100644 --- a/integration/service/spatialindexer/s2indexer/builder_test.go +++ b/integration/service/spatialindexer/s2indexer/builder_test.go @@ -2,6 +2,7 @@ package s2indexer import ( "reflect" + "sort" "testing" "github.com/Telenav/osrm-backend/integration/service/spatialindexer" @@ -253,6 +254,14 @@ func TestBuild(t *testing.T) { for i, c := range cases { actual := build(c.points, 6, 30) + + // sort slice to directly use DeepEqual, Build()'s result no need to guarantee the sequence + for k := range actual { + sort.Slice(actual[k], func(i, j int) bool { + return actual[k][i] < actual[k][j] + }) + } + if !reflect.DeepEqual(actual, c.expect) { t.Errorf("parse case %d \n%v, expect\n %v \n but got %v", i, c, c.expect, actual) } From 82705722b2c32a9dee5445a8957f5751cd98a4d4 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Thu, 26 Mar 2020 11:11:11 -0700 Subject: [PATCH 8/8] fix: adjust code based on review. --- .../chargestations-connectivity-gen/main.go | 1 + integration/service/spatialindexer/doc.go | 2 +- .../{iterface.go => interface.go} | 2 +- .../{ => spatialindexer}/poiloader/loader.go | 0 .../poiloader/poi_format.go | 6 +++ .../poiloader/sample_input.json | 0 .../spatialindexer/s2indexer/builder.go | 3 ++ .../spatialindexer/s2indexer/dumper.go | 50 +++++++++++-------- .../spatialindexer/s2indexer/dumper_test.go | 4 +- .../spatialindexer/s2indexer/indexer.go | 28 ++++++----- .../spatialindexer/s2indexer/spatial_query.go | 10 +++- .../s2indexer/spatial_query_test.go | 2 +- 12 files changed, 68 insertions(+), 40 deletions(-) rename integration/service/spatialindexer/{iterface.go => interface.go} (93%) rename integration/service/{ => spatialindexer}/poiloader/loader.go (100%) rename integration/service/{ => spatialindexer}/poiloader/poi_format.go (84%) rename integration/service/{ => spatialindexer}/poiloader/sample_input.json (100%) diff --git a/integration/cmd/chargestations-connectivity-gen/main.go b/integration/cmd/chargestations-connectivity-gen/main.go index 934fe5d5111..bd1acb5d084 100644 --- a/integration/cmd/chargestations-connectivity-gen/main.go +++ b/integration/cmd/chargestations-connectivity-gen/main.go @@ -5,5 +5,6 @@ import ( ) func main() { + // @todo: add logic to generate connectivity for charge stations fmt.Print("Hello World!") } diff --git a/integration/service/spatialindexer/doc.go b/integration/service/spatialindexer/doc.go index 3e848cc3a20..87b49a73af4 100644 --- a/integration/service/spatialindexer/doc.go +++ b/integration/service/spatialindexer/doc.go @@ -9,7 +9,7 @@ // } // // -// Sample Scenario 2: Dump s2Indexer's content to folder +// Sample Scenario 2: Dump S2Indexer's content to folder // indexer.Dump(folderPath) // // diff --git a/integration/service/spatialindexer/iterface.go b/integration/service/spatialindexer/interface.go similarity index 93% rename from integration/service/spatialindexer/iterface.go rename to integration/service/spatialindexer/interface.go index dfcca077f7c..7d0aaf5138c 100644 --- a/integration/service/spatialindexer/iterface.go +++ b/integration/service/spatialindexer/interface.go @@ -26,7 +26,7 @@ type PointID int64 // Finder answers special query type Finder interface { - // FindNearByIDs returns a group of points near to given center location + // FindNearByPointIDs returns a group of points near to given center location FindNearByPointIDs(center Location, radius float64, limitCount int) []PointInfo } diff --git a/integration/service/poiloader/loader.go b/integration/service/spatialindexer/poiloader/loader.go similarity index 100% rename from integration/service/poiloader/loader.go rename to integration/service/spatialindexer/poiloader/loader.go diff --git a/integration/service/poiloader/poi_format.go b/integration/service/spatialindexer/poiloader/poi_format.go similarity index 84% rename from integration/service/poiloader/poi_format.go rename to integration/service/spatialindexer/poiloader/poi_format.go index 95c3b5e4055..746a7456555 100644 --- a/integration/service/poiloader/poi_format.go +++ b/integration/service/spatialindexer/poiloader/poi_format.go @@ -1,5 +1,6 @@ package poiloader +// Element represent information loaded for point record type Element struct { ID int64 `json:"id"` VendorCode VendorCode `json:"vendor_code"` @@ -32,24 +33,28 @@ type Element struct { Amenity Amenity `json:"amenity"` } +// Amenity is defined by OSM format: https://wiki.openstreetmap.org/wiki/Tag:amenity%3Dcharging_station type Amenity string const ( ChargingStation Amenity = "charging_station" ) +// Country records IOS code type Country string const ( Usa Country = "USA" ) +// SpaceID is used to define a code for political defined administrative area type SpaceID string const ( UsaCA SpaceID = "USA_CA" ) +// VendorCode defines data source provider type VendorCode string const ( @@ -58,6 +63,7 @@ const ( Nolp VendorCode = "NOLP" ) +// CategoryIDGather records aggregate data for certain type of point type CategoryIDGather struct { Integer *int64 String *string diff --git a/integration/service/poiloader/sample_input.json b/integration/service/spatialindexer/poiloader/sample_input.json similarity index 100% rename from integration/service/poiloader/sample_input.json rename to integration/service/spatialindexer/poiloader/sample_input.json diff --git a/integration/service/spatialindexer/s2indexer/builder.go b/integration/service/spatialindexer/s2indexer/builder.go index eed79895dbb..49550b7ef47 100644 --- a/integration/service/spatialindexer/s2indexer/builder.go +++ b/integration/service/spatialindexer/s2indexer/builder.go @@ -28,9 +28,11 @@ func build(points []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.Cel if i > maxLevel { continue } + parentCellID := leafCellID.Parent(i) cellIDs = append(cellIDs, parentCellID) } + pointID2CellIDs[p.ID] = cellIDs } @@ -40,6 +42,7 @@ func build(points []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.Cel var pointIDs []spatialindexer.PointID cellID2PointIDs[cellID] = pointIDs } + cellID2PointIDs[cellID] = append(cellID2PointIDs[cellID], pointID) } } diff --git a/integration/service/spatialindexer/s2indexer/dumper.go b/integration/service/spatialindexer/s2indexer/dumper.go index d70fa978785..0432a6d5a4b 100644 --- a/integration/service/spatialindexer/s2indexer/dumper.go +++ b/integration/service/spatialindexer/s2indexer/dumper.go @@ -14,9 +14,9 @@ import ( const cellID2PointIDsFileName = "cellID2PointIDs.gob" const pointID2LocationFileName = "pointID2Location.gob" -func serializeS2Indexer(indexer *s2Indexer, folderPath string) error { +func serializeS2Indexer(indexer *S2Indexer, folderPath string) error { if !strings.HasSuffix(folderPath, api.Slash) { - folderPath = folderPath + "/" + folderPath += "/" } if err := serializeCellID2PointIDs(indexer, folderPath); err != nil { @@ -27,45 +27,51 @@ func serializeS2Indexer(indexer *s2Indexer, folderPath string) error { return err } - glog.Infof("Successfully serialize s2Indexer to folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", + glog.Infof("Successfully serialize S2Indexer to folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", folderPath, len(indexer.cellID2PointIDs), len(indexer.pointID2Location)) return nil } -func serializeCellID2PointIDs(indexer *s2Indexer, folderPath string) error { +func serializeCellID2PointIDs(indexer *S2Indexer, folderPath string) error { buf := new(bytes.Buffer) encoder := gob.NewEncoder(buf) err := encoder.Encode(indexer.cellID2PointIDs) + if err != nil { - glog.Errorf("During encode s2Indexer's cellID2PointIDs met error %v", err) + glog.Errorf("During encode S2Indexer's cellID2PointIDs met error %v", err) return err } + if err = ioutil.WriteFile(folderPath+cellID2PointIDsFileName, buf.Bytes(), 0644); err != nil { - glog.Errorf("During dump s2Indexer's cellID2PointIDs met error %v", err) + glog.Errorf("During dump S2Indexer's cellID2PointIDs met error %v", err) return err } + return nil } -func serializePointID2Location(indexer *s2Indexer, folderPath string) error { +func serializePointID2Location(indexer *S2Indexer, folderPath string) error { buf := new(bytes.Buffer) encoder := gob.NewEncoder(buf) err := encoder.Encode(indexer.pointID2Location) + if err != nil { - glog.Errorf("During encode s2Indexer's pointID2Location met error %v", err) + glog.Errorf("During encode S2Indexer's pointID2Location met error %v", err) return err } + if err = ioutil.WriteFile(folderPath+pointID2LocationFileName, buf.Bytes(), 0644); err != nil { - glog.Errorf("During dump s2Indexer's pointID2Location met error %v", err) + glog.Errorf("During dump S2Indexer's pointID2Location met error %v", err) return err } + return nil } -func deSerializeS2Indexer(indexer *s2Indexer, folderPath string) error { +func deSerializeS2Indexer(indexer *S2Indexer, folderPath string) error { if !strings.HasSuffix(folderPath, api.Slash) { - folderPath = folderPath + "/" + folderPath += "/" } if err := deSerializeCellID2PointIDs(indexer, folderPath); err != nil { @@ -76,49 +82,53 @@ func deSerializeS2Indexer(indexer *s2Indexer, folderPath string) error { return err } - glog.Infof("Successfully deserialize s2Indexer from folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", + glog.Infof("Successfully deserialize S2Indexer from folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", folderPath, len(indexer.cellID2PointIDs), len(indexer.pointID2Location)) return nil } -func deSerializeCellID2PointIDs(indexer *s2Indexer, folderPath string) error { +func deSerializeCellID2PointIDs(indexer *S2Indexer, folderPath string) error { byteArray, err := ioutil.ReadFile(folderPath + cellID2PointIDsFileName) if err != nil { - glog.Errorf("During load s2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) + glog.Errorf("During load S2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) return err } + buf := bytes.NewBuffer(byteArray) decoder := gob.NewDecoder(buf) err = decoder.Decode(&indexer.cellID2PointIDs) + if err != nil { - glog.Errorf("During decode s2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) + glog.Errorf("During decode S2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) return err } return nil } -func deSerializePointID2Location(indexer *s2Indexer, folderPath string) error { +func deSerializePointID2Location(indexer *S2Indexer, folderPath string) error { byteArray, err := ioutil.ReadFile(folderPath + pointID2LocationFileName) if err != nil { - glog.Errorf("During load s2Indexer's pointID2Location from %s met error %v", folderPath, err) + glog.Errorf("During load S2Indexer's pointID2Location from %s met error %v", folderPath, err) return err } + buf := bytes.NewBuffer(byteArray) decoder := gob.NewDecoder(buf) err = decoder.Decode(&indexer.pointID2Location) + if err != nil { - glog.Errorf("During decode s2Indexer's pointID2Location from %s met error %v", folderPath, err) + glog.Errorf("During decode S2Indexer's pointID2Location from %s met error %v", folderPath, err) return err } - return nil + return nil } func removeAllDumpFiles(folderPath string) error { if !strings.HasSuffix(folderPath, api.Slash) { - folderPath = folderPath + "/" + folderPath += "/" } _, err := os.Stat(folderPath + cellID2PointIDsFileName) diff --git a/integration/service/spatialindexer/s2indexer/dumper_test.go b/integration/service/spatialindexer/s2indexer/dumper_test.go index a8393dfed05..5fd860dae76 100644 --- a/integration/service/spatialindexer/s2indexer/dumper_test.go +++ b/integration/service/spatialindexer/s2indexer/dumper_test.go @@ -10,8 +10,8 @@ import ( ) func TestDumper(t *testing.T) { - cases := []s2Indexer{ - s2Indexer{ + cases := []S2Indexer{ + S2Indexer{ cellID2PointIDs: map[s2.CellID][]spatialindexer.PointID{ 9263834064756932608: {1, 2, 3}, // 4/0010133 9263851656942977024: {1}, // 4/00101332 diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go index 2f524529d0d..6806555078c 100644 --- a/integration/service/spatialindexer/s2indexer/indexer.go +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -1,30 +1,32 @@ package s2indexer import ( - "github.com/Telenav/osrm-backend/integration/service/poiloader" "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer/poiloader" "github.com/golang/geo/s2" "github.com/golang/glog" ) -type s2Indexer struct { +// S2Indexer provide spatial index ability based on google s2 +type S2Indexer struct { cellID2PointIDs map[s2.CellID][]spatialindexer.PointID pointID2Location map[spatialindexer.PointID]spatialindexer.Location } // NewS2Indexer generates spatial indexer based on google s2 -func NewS2Indexer() *s2Indexer { - return &s2Indexer{} +func NewS2Indexer() *S2Indexer { + return &S2Indexer{} } // Build constructs S2 indexer -func (indexer *s2Indexer) Build(filePath string) *s2Indexer { +func (indexer *S2Indexer) Build(filePath string) *S2Indexer { records, err := poiloader.LoadData(filePath) if err != nil || len(records) == 0 { return nil } var pointInfos []spatialindexer.PointInfo + for _, record := range records { pointInfo := spatialindexer.PointInfo{ ID: elementID2PointID(record.ID), @@ -45,28 +47,28 @@ func (indexer *s2Indexer) Build(filePath string) *s2Indexer { return indexer } -// Load s2Indexer's data from contents recorded in folder -func (indexer *s2Indexer) Load(folderPath string) *s2Indexer { +// Load S2Indexer's data from contents recorded in folder +func (indexer *S2Indexer) Load(folderPath string) *S2Indexer { if err := deSerializeS2Indexer(indexer, folderPath); err != nil { - glog.Errorf("Load s2Indexer's data from folder %s failed, err=%v\n", folderPath, err) + glog.Errorf("Load S2Indexer's data from folder %s failed, err=%v\n", folderPath, err) return nil } return indexer } -// Dump s2Indexer's content into folderPath -func (indexer *s2Indexer) Dump(folderPath string) { +// Dump S2Indexer's content into folderPath +func (indexer *S2Indexer) Dump(folderPath string) { if err := serializeS2Indexer(indexer, folderPath); err != nil { - glog.Errorf("Dump s2Indexer's data to folder %s failed, err=%v\n", folderPath, err) + glog.Errorf("Dump S2Indexer's data to folder %s failed, err=%v\n", folderPath, err) } } -func (indexer s2Indexer) getPointLocationByPointID(id spatialindexer.PointID) (spatialindexer.Location, bool) { +func (indexer S2Indexer) getPointLocationByPointID(id spatialindexer.PointID) (spatialindexer.Location, bool) { location, ok := indexer.pointID2Location[id] return location, ok } -func (indexer s2Indexer) getPointIDsByS2CellID(cellid s2.CellID) ([]spatialindexer.PointID, bool) { +func (indexer S2Indexer) getPointIDsByS2CellID(cellid s2.CellID) ([]spatialindexer.PointID, bool) { pointIDs, ok := indexer.cellID2PointIDs[cellid] return pointIDs, ok } diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go index 0cd26b65d75..2b7c5bddafb 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -20,12 +20,15 @@ func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) [ radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) region := s2.Region(s2.CapFromCenterAngle(center, radius)) cellUnion := regionCover.Covering(region) + return ([]s2.CellID)(cellUnion) } -func queryNearByPoints(indexer *s2Indexer, point spatialindexer.Location, radius float64) []spatialindexer.PointInfo { +func queryNearByPoints(indexer *S2Indexer, point spatialindexer.Location, radius float64) []spatialindexer.PointInfo { var result []spatialindexer.PointInfo + cellIDs := queryNearByS2Cells(point, radius) + for _, cellID := range cellIDs { pointIDs, hasCellID := indexer.getPointIDsByS2CellID(cellID) if !hasCellID { @@ -35,7 +38,7 @@ func queryNearByPoints(indexer *s2Indexer, point spatialindexer.Location, radius for _, pointID := range pointIDs { location, hasPointID := indexer.getPointLocationByPointID(pointID) if !hasPointID { - glog.Errorf("In queryNearByPoints, use incorrect pointID %v to query s2Indexer\n", pointID) + glog.Errorf("In queryNearByPoints, use incorrect pointID %v to query S2Indexer\n", pointID) continue } @@ -52,10 +55,12 @@ func queryNearByPoints(indexer *s2Indexer, point spatialindexer.Location, radius func generateDebugInfo4CellIDs(cellIDs []s2.CellID) { glog.Info("=================================\n") glog.Info("generateDebugInfo4CellIDs\n") + for _, cellID := range cellIDs { glog.Infof("CellID value = %d(uint64), string = %v, token = %s, level = %d\n", (uint64)(cellID), cellID, cellID.ToToken(), cellID.Level()) } + glog.Info("=================================\n") } @@ -71,5 +76,6 @@ func generateDebugURL(cellIDs []s2.CellID) string { for _, cellID := range cellIDs { url += cellID.ToToken() + "," } + return url } diff --git a/integration/service/spatialindexer/s2indexer/spatial_query_test.go b/integration/service/spatialindexer/s2indexer/spatial_query_test.go index 2b9599610a0..8926bc11845 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query_test.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query_test.go @@ -10,7 +10,7 @@ import ( ) func TestSpatialIndexQuery1(t *testing.T) { - fakeIndexer1 := s2Indexer{ + fakeIndexer1 := S2Indexer{ cellID2PointIDs: map[s2.CellID][]spatialindexer.PointID{ 9263834064756932608: {1, 2}, // 4/0010133 9263851656942977024: {1, 2}, // 4/00101332