From 8061068b7a691c6e9f8d570b9ce221fa2a8b7a36 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 16:50:50 -0700 Subject: [PATCH 01/27] feat: implement spartial index query based on google::s2 issue: https://github.com/Telenav/osrm-backend/issues/239 --- .../chargestations-connectivity-gen/doc.go | 17 ++ integration/go.mod | 1 + integration/go.sum | 3 + integration/service/poiloader/loader.go | 26 ++ integration/service/poiloader/poi_format.go | 64 +++++ .../service/poiloader/sample_input.json | 64 +++++ integration/service/spatialindexer/doc.go | 19 ++ .../service/spatialindexer/iterface.go | 30 ++ .../spatialindexer/s2indexer/builder.go | 48 ++++ .../spatialindexer/s2indexer/builder_test.go | 260 ++++++++++++++++++ .../spatialindexer/s2indexer/dumper.go | 146 ++++++++++ .../spatialindexer/s2indexer/dumper_test.go | 65 +++++ .../spatialindexer/s2indexer/indexer.go | 76 +++++ .../spatialindexer/s2indexer/spatial_query.go | 76 +++++ .../s2indexer/spatial_query_test.go | 245 +++++++++++++++++ 15 files changed, 1140 insertions(+) create mode 100644 integration/cmd/chargestations-connectivity-gen/doc.go create mode 100644 integration/service/poiloader/loader.go create mode 100644 integration/service/poiloader/poi_format.go create mode 100644 integration/service/poiloader/sample_input.json create mode 100644 integration/service/spatialindexer/doc.go create mode 100644 integration/service/spatialindexer/iterface.go create mode 100644 integration/service/spatialindexer/s2indexer/builder.go create mode 100644 integration/service/spatialindexer/s2indexer/builder_test.go create mode 100644 integration/service/spatialindexer/s2indexer/dumper.go create mode 100644 integration/service/spatialindexer/s2indexer/dumper_test.go create mode 100644 integration/service/spatialindexer/s2indexer/indexer.go create mode 100644 integration/service/spatialindexer/s2indexer/spatial_query.go create mode 100644 integration/service/spatialindexer/s2indexer/spatial_query_test.go diff --git a/integration/cmd/chargestations-connectivity-gen/doc.go b/integration/cmd/chargestations-connectivity-gen/doc.go new file mode 100644 index 00000000000..8e0f8b31e38 --- /dev/null +++ b/integration/cmd/chargestations-connectivity-gen/doc.go @@ -0,0 +1,17 @@ +// package main contains the tool of chargestation-connectivity generator +// stage 1: +// inputs is json file +// => convert to slice of [id:string,location: lat,lon] +// => calculate cellids for each point(for all levels) +// => build revese index for cellid -> ids +// stage 2: +// => iterate each point +// => generate a circle(s2::cap), find all cellids intersect with that circle +// => retrieve all ids +// => generate result of id(from), ids(all ids in certain distance) +// stage 3: +// => load data from file +// => for each line, its formid and all other ids +// => calculate distance between fromid and all other ids +// => sort result based on distance and write back to file +package main diff --git a/integration/go.mod b/integration/go.mod index bc2c88727a9..415bc0084aa 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -3,6 +3,7 @@ module github.com/Telenav/osrm-backend/integration go 1.13 require ( + github.com/golang/geo v0.0.0-20200319012246-673a6f80352d github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.1 diff --git a/integration/go.sum b/integration/go.sum index 8bb21f4ae9f..6c3a3f88da6 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1,6 +1,9 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Telenav/osrm-backend v5.22.0+incompatible h1:v5K4VmdxyQjFXN20ZGmk9MchGTHWWIMUXiXmAndNvlQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/golang/geo v0.0.0-20200319012246-673a6f80352d h1:C/hKUcHT483btRbeGkrRjJz+Zbcj8audldIi9tRJDCc= +github.com/golang/geo v0.0.0-20200319012246-673a6f80352d/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= diff --git a/integration/service/poiloader/loader.go b/integration/service/poiloader/loader.go new file mode 100644 index 00000000000..5f86e98bfa1 --- /dev/null +++ b/integration/service/poiloader/loader.go @@ -0,0 +1,26 @@ +package poiloader + +import ( + "encoding/json" + "io/ioutil" + + "github.com/golang/glog" +) + +// LoadPOI accepts json file recorded with poi data and returns deserialized result +func LoadData(filePath string) ([]Element, error) { + var elements []Element + + file, err := ioutil.ReadFile(filePath) + if err != nil { + glog.Errorf("While load file %s, met error %v\n", filePath, err) + return elements, err + } + + err = json.Unmarshal(file, &elements) + if err != nil { + glog.Errorf("While unmarshal json file %s, met error %v\n", filePath, err) + return elements, err + } + return elements, nil +} diff --git a/integration/service/poiloader/poi_format.go b/integration/service/poiloader/poi_format.go new file mode 100644 index 00000000000..95c3b5e4055 --- /dev/null +++ b/integration/service/poiloader/poi_format.go @@ -0,0 +1,64 @@ +package poiloader + +type Element struct { + ID int64 `json:"id"` + VendorCode VendorCode `json:"vendor_code"` + VendorPoiID string `json:"vendor_poi_id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + NavLat float64 `json:"nav_lat"` + NavLon float64 `json:"nav_lon"` + DisLat float64 `json:"dis_lat"` + DisLon float64 `json:"dis_lon"` + MapLinkID int64 `json:"map_link_id"` + SideOfStreet string `json:"side_of_street"` + Country Country `json:"country"` + SpaceID SpaceID `json:"space_id"` + AirportCode string `json:"airport_code"` + IsNational bool `json:"is_national"` + IsStateImportance bool `json:"is_state_importance"` + IsCityImportance bool `json:"is_city_importance"` + Fax string `json:"fax"` + Phone *CategoryIDGather `json:"phone"` + EncodedPhone string `json:"encoded_phone"` + Email string `json:"email"` + WebURL string `json:"web_url"` + CategoryIDGather *CategoryIDGather `json:"category_id_gather"` + ChainGather string `json:"chain_gather"` + RawCategoryGather string `json:"raw_category_gather"` + ChildGather string `json:"child_gather"` + ParentGather string `json:"parent_gather"` + Hilbert float64 `json:"hilbert"` + Amenity Amenity `json:"amenity"` +} + +type Amenity string + +const ( + ChargingStation Amenity = "charging_station" +) + +type Country string + +const ( + Usa Country = "USA" +) + +type SpaceID string + +const ( + UsaCA SpaceID = "USA_CA" +) + +type VendorCode string + +const ( + Noel VendorCode = "NOEL" + Noft VendorCode = "NOFT" + Nolp VendorCode = "NOLP" +) + +type CategoryIDGather struct { + Integer *int64 + String *string +} diff --git a/integration/service/poiloader/sample_input.json b/integration/service/poiloader/sample_input.json new file mode 100644 index 00000000000..6f83a702a3b --- /dev/null +++ b/integration/service/poiloader/sample_input.json @@ -0,0 +1,64 @@ +[ + { + "id":12345, + "vendor_code":"NOFT", + "vendor_poi_id":"NOFT-10_NREL99930@-@NOCO-37145190@-@NOCO-996595530", + "lat":0.48651, + "lon":0.66526, + "nav_lat":0.48651, + "nav_lon":0.66526, + "dis_lat":0.48651, + "dis_lon":0.66526, + "map_link_id":12345, + "side_of_street":"+", + "country":"USA", + "space_id":"USA_CA", + "airport_code":"NULL", + "is_national":false, + "is_state_importance":false, + "is_city_importance":false, + "fax":"NULL", + "phone":123456789, + "encoded_phone":"123456789", + "email":"NULL", + "web_url":"NULL", + "category_id_gather":"771``794``915", + "chain_gather":"1364@-@@-@", + "raw_category_gather":"NT@-@3578``NT@-@6000``NAICS@-@522110``NT@-@5540", + "child_gather":"NULL", + "parent_gather":"1718449@-@1``2216423@-@1", + "hilbert":2.32E+11, + "amenity":"charging_station" + }, + { + "id":12346, + "vendor_code":"NOFT", + "vendor_poi_id":"NOFT-10_NREL99930@-@NOCO-37145190@-@NOCO-996595530", + "lat":0.48651, + "lon":0.66526, + "nav_lat":0.48651, + "nav_lon":0.66526, + "dis_lat":0.48651, + "dis_lon":0.66526, + "map_link_id":12345, + "side_of_street":"+", + "country":"USA", + "space_id":"USA_CA", + "airport_code":"NULL", + "is_national":false, + "is_state_importance":false, + "is_city_importance":false, + "fax":"NULL", + "phone":123456789, + "encoded_phone":"123456789", + "email":"NULL", + "web_url":"NULL", + "category_id_gather":"771``794``915", + "chain_gather":"1364@-@@-@", + "raw_category_gather":"NT@-@3578``NT@-@6000``NAICS@-@522110``NT@-@5540", + "child_gather":"NULL", + "parent_gather":"1718449@-@1``2216423@-@1", + "hilbert":2.32E+11, + "amenity":"charging_station" + } +] \ No newline at end of file diff --git a/integration/service/spatialindexer/doc.go b/integration/service/spatialindexer/doc.go new file mode 100644 index 00000000000..2e98a65aaeb --- /dev/null +++ b/integration/service/spatialindexer/doc.go @@ -0,0 +1,19 @@ +// package spatialindexer answers query of nearest pois for given point +// +// Sample 1: Build connectivity for charge stations during pro-processing +// indexer := NewS2Indexer().Build(poiCsvFile) +// for _, stationPoint := range chargeStations { +// nearbyStations := indexer.FindNearByIDs(stationPoint, 800km, -1) +// rankedStations := indexer.RankingIDsByShortestDistance(stationPoint, nearbyStations) +// } +// +// +// Sample 2: Dump s2Indexer's content to folder +// indexer.Dump(folderPath) +// +// +// Sample 3: Query reachable charge stations with current energy level +// indexer := NewS2Indexer().Load(folderPath) +// nearbyStations := indexer.FindNearByIDs(currentPoint, currentEnergyLevel, -1) +// +package spatialindexer diff --git a/integration/service/spatialindexer/iterface.go b/integration/service/spatialindexer/iterface.go new file mode 100644 index 00000000000..e8213635271 --- /dev/null +++ b/integration/service/spatialindexer/iterface.go @@ -0,0 +1,30 @@ +package spatialindexer + +// Location for poi point +// @todo: will be replaced by the one in map +type Location struct { + Latitude float64 + Longitude float64 +} + +type PointInfo struct { + ID PointID + Location Location +} + +type RankedPointInfo struct { + PointInfo + Distance float64 +} + +type PointID int64 + +type Finder interface { + FindNearByIDs(center Location, radius float64, limitCount int) []PointInfo +} + +type Ranker interface { + RankingIDsByGreatCircleDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo + + RankingIDsByShortestDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo +} diff --git a/integration/service/spatialindexer/s2indexer/builder.go b/integration/service/spatialindexer/s2indexer/builder.go new file mode 100644 index 00000000000..c7ef012d7bf --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/builder.go @@ -0,0 +1,48 @@ +package s2indexer + +import ( + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" +) + +// https://s2geometry.io/resources/s2cell_statistics.html +// Level = 6 means average area size is 20754.64km2 +// Level = 9 means average area size is 324.29km2 +const minS2Level = 9 + +// Level = 19 means average area size is 309.27km2 +// Level = 20 means average area size is 77.32km2 +const maxS2Level = 20 + +func build(pois []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.CellID][]spatialindexer.PointID { + pointID2CellIDs := make(map[spatialindexer.PointID][]s2.CellID) + cellID2PointIDs := make(map[s2.CellID][]spatialindexer.PointID) + + for _, p := range pois { + leafCellID := s2.CellFromLatLng(s2.LatLngFromDegrees(p.Location.Latitude, p.Location.Longitude)).ID() + + var cellIDs []s2.CellID + // For level = 30, its parent equal to current + // So no need append leafCellID outside of for loop if maxLevel = 30 + for i := leafCellID.Level(); i > minLevel; i-- { + if i > maxLevel { + continue + } + parentCellID := leafCellID.Parent(i) + cellIDs = append(cellIDs, parentCellID) + } + pointID2CellIDs[p.ID] = cellIDs + } + + for pointID, cellIDs := range pointID2CellIDs { + for _, cellID := range cellIDs { + if _, ok := cellID2PointIDs[cellID]; !ok { + var pointIDs []spatialindexer.PointID + cellID2PointIDs[cellID] = pointIDs + } + cellID2PointIDs[cellID] = append(cellID2PointIDs[cellID], pointID) + } + } + + return cellID2PointIDs +} diff --git a/integration/service/spatialindexer/s2indexer/builder_test.go b/integration/service/spatialindexer/s2indexer/builder_test.go new file mode 100644 index 00000000000..4fa3c8e3847 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/builder_test.go @@ -0,0 +1,260 @@ +package s2indexer + +import ( + "reflect" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" +) + +// online tool for google s2: http://s2map.com/#order=latlng&mode=polygon&s2=false&points= +func TestBuild(t *testing.T) { + cases := []struct { + points []spatialindexer.PointInfo + expect map[s2.CellID][]spatialindexer.PointID + }{ + { + []spatialindexer.PointInfo{ + spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Latitude: 37.402701, + Longitude: -121.974096, + }, + }, + }, + map[s2.CellID][]spatialindexer.PointID{ + 9263834064756932608: {1}, // 4/0010133 + 9263851656942977024: {1}, // 4/00101332 + 9263847258896465920: {1}, // 4/001013321 + 9263843960361582592: {1}, // 4/0010133210 + 9263844784995303424: {1}, // 4/00101332103 + 9263844853714780160: {1}, // 4/001013321032 + 9263844836534910976: {1}, // 4/0010133210321 + 9263844823650009088: {1}, // 4/00101332103210 + 9263844822576267264: {1}, // 4/001013321032101 + 9263844823381573632: {1}, // 4/0010133210321013 + 9263844823180247040: {1}, // 4/00101332103210130 + 9263844823129915392: {1}, // 4/001013321032101300 + 9263844823134109696: {1}, // 4/0010133210321013002 + 9263844823130963968: {1}, // 4/00101332103210130020 + 9263844823130701824: {1}, // 4/001013321032101300201 + 9263844823130898432: {1}, // 4/0010133210321013002013 + 9263844823130947584: {1}, // 4/00101332103210130020133 + 9263844823130943488: {1}, // 4/001013321032101300201331 + 9263844823130944512: {1}, // 4/0010133210321013002013312 + 9263844823130943744: {1}, // 4/00101332103210130020133120 + 9263844823130943680: {1}, // 4/001013321032101300201331201 + 9263844823130943696: {1}, // 4/0010133210321013002013312012 + 9263844823130943708: {1}, // 4/00101332103210130020133120123 + 9263844823130943709: {1}, // 4/001013321032101300201331201232 + }, + }, + + // Test case of 2 points, both near to 4655 Great America Pkwy + { + []spatialindexer.PointInfo{ + spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Latitude: 37.402701, + Longitude: -121.974096, + }, + }, + spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Latitude: 37.403530, + Longitude: -121.969768, + }, + }, + }, + map[s2.CellID][]spatialindexer.PointID{ + 9263834064756932608: {1, 2}, // 4/0010133 + 9263851656942977024: {1, 2}, // 4/00101332 + 9263847258896465920: {1, 2}, // 4/001013321 + 9263843960361582592: {1, 2}, // 4/0010133210 + 9263844784995303424: {1, 2}, // 4/00101332103 + 9263844716275826688: {2}, // 4/001013321031 + 9263844733455695872: {2}, // 4/0010133210312 + 9263844746340597760: {2}, // 4/00101332103123 + 9263844749561823232: {2}, // 4/001013321031233 + 9263844749830258688: {2}, // 4/0010133210312332 + 9263844750031585280: {2}, // 4/00101332103123323 + 9263844749981253632: {2}, // 4/001013321031233230 + 9263844749993836544: {2}, // 4/0010133210312332303 + 9263844749996982272: {2}, // 4/00101332103123323033 + 9263844749996720128: {2}, // 4/001013321031233230331 + 9263844749996916736: {2}, // 4/0010133210312332303313 + 9263844749996900352: {2}, // 4/00101332103123323033131 + 9263844749996896256: {2}, // 4/001013321031233230331311 + 9263844749996893184: {2}, // 4/0010133210312332303313110 + 9263844749996892416: {2}, // 4/00101332103123323033131100 + 9263844749996892608: {2}, // 4/001013321031233230331311003 + 9263844749996892592: {2}, // 4/0010133210312332303313110031 + 9263844749996892588: {2}, // 4/00101332103123323033131100311 + 9263844749996892591: {2}, // 4/001013321031233230331311003113 + 9263844853714780160: {1}, // 4/001013321032 + 9263844836534910976: {1}, // 4/0010133210321 + 9263844823650009088: {1}, // 4/00101332103210 + 9263844822576267264: {1}, // 4/001013321032101 + 9263844823381573632: {1}, // 4/0010133210321013 + 9263844823180247040: {1}, // 4/00101332103210130 + 9263844823129915392: {1}, // 4/001013321032101300 + 9263844823134109696: {1}, // 4/0010133210321013002 + 9263844823130963968: {1}, // 4/00101332103210130020 + 9263844823130701824: {1}, // 4/001013321032101300201 + 9263844823130898432: {1}, // 4/0010133210321013002013 + 9263844823130947584: {1}, // 4/00101332103210130020133 + 9263844823130943488: {1}, // 4/001013321032101300201331 + 9263844823130944512: {1}, // 4/0010133210321013002013312 + 9263844823130943744: {1}, // 4/00101332103210130020133120 + 9263844823130943680: {1}, // 4/001013321032101300201331201 + 9263844823130943696: {1}, // 4/0010133210321013002013312012 + 9263844823130943708: {1}, // 4/00101332103210130020133120123 + 9263844823130943709: {1}, // 4/001013321032101300201331201232 + }, + }, + + // Test case of 4 points, points distrubted in CA, US + // Each points have about 100km distance with each other + { + []spatialindexer.PointInfo{ + spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Latitude: 37.651275, + Longitude: -122.413744, + }, + }, + spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Latitude: 36.776215, + Longitude: -121.733663, + }, + }, + spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Latitude: 36.122438, + Longitude: -121.022936, + }, + }, + spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Latitude: 35.365543, + Longitude: -120.850000, + }, + }, + }, + map[s2.CellID][]spatialindexer.PointID{ + 9263411852291866624: {2}, // 4/0010130 + 9263359075733733376: {2}, // 4/00101300 + 9263345881594200064: {2}, // 4/001013000 + 9263349180129083392: {2}, // 4/0010130003 + 9263349455006990336: {2}, // 4/00101300032 + 9263349386287513600: {2}, // 4/001013000321 + 9263349437827121152: {2}, // 4/0010130003213 + 9263349433532153856: {2}, // 4/00101300032131 + 9263349432458412032: {2}, // 4/001013000321311 + 9263349432726847488: {2}, // 4/0010130003213112 + 9263349432928174080: {2}, // 4/00101300032131123 + 9263349432877842432: {2}, // 4/001013000321311230 + 9263349432873648128: {2}, // 4/0010130003213112301 + 9263349432874696704: {2}, // 4/00101300032131123012 + 9263349432874434560: {2}, // 4/001013000321311230121 + 9263349432874500096: {2}, // 4/0010130003213112301212 + 9263349432874516480: {2}, // 4/00101300032131123012122 + 9263349432874528768: {2}, // 4/001013000321311230121223 + 9263349432874525696: {2}, // 4/0010130003213112301212230 + 9263349432874526464: {2}, // 4/00101300032131123012122303 + 9263349432874526528: {2}, // 4/001013000321311230121223032 + 9263349432874526544: {2}, // 4/0010130003213112301212230322 + 9263349432874526556: {2}, // 4/00101300032131123012122303223 + 9263349432874526559: {2}, // 4/001013000321311230121223032233 + 9263693327268577280: {1}, // 4/0010132 + 9263746103826710528: {1}, // 4/00101323 + 9263759297966243840: {1}, // 4/001013233 + 9263755999431360512: {1}, // 4/0010132330 + 9263756824065081344: {1}, // 4/00101323303 + 9263756755345604608: {1}, // 4/001013233031 + 9263756703805997056: {1}, // 4/0010132330310 + 9263756708100964352: {1}, // 4/00101323303102 + 9263756709174706176: {1}, // 4/001013233031022 + 9263756709980012544: {1}, // 4/0010132330310223 + 9263756709912903680: {1}, // 4/00101323303102231 + 9263756709862572032: {1}, // 4/001013233031022310 + 9263756709866766336: {1}, // 4/0010132330310223102 + 9263756709867814912: {1}, // 4/00101323303102231022 + 9263756709867028480: {1}, // 4/001013233031022310220 + 9263756709866962944: {1}, // 4/0010132330310223102201 + 9263756709866913792: {1}, // 4/00101323303102231022010 + 9263756709866909696: {1}, // 4/001013233031022310220101 + 9263756709866908672: {1}, // 4/0010132330310223102201011 + 9263756709866907904: {1}, // 4/00101323303102231022010110 + 9263756709866907840: {1}, // 4/001013233031022310220101101 + 9263756709866907856: {1}, // 4/0010132330310223102201011012 + 9263756709866907844: {1}, // 4/00101323303102231022010110120 + 9263756709866907845: {1}, // 4/001013233031022310220101101202 + 9264678489687064576: {3}, // 4/0010211 + 9264731266245197824: {3}, // 4/00102113 + 9264718072105664512: {3}, // 4/001021130 + 9264719171617292288: {3}, // 4/0010211302 + 9264718346983571456: {3}, // 4/00102113020 + 9264718415703048192: {3}, // 4/001021130202 + 9264718398523179008: {3}, // 4/0010211302021 + 9264718411408080896: {3}, // 4/00102113020213 + 9264718414629306368: {3}, // 4/001021130202133 + 9264718413824000000: {3}, // 4/0010211302021330 + 9264718414025326592: {3}, // 4/00102113020213303 + 9264718414042103808: {3}, // 4/001021130202133032 + 9264718414046298112: {3}, // 4/0010211302021330322 + 9264718414043152384: {3}, // 4/00102113020213303220 + 9264718414042365952: {3}, // 4/001021130202133032200 + 9264718414042169344: {3}, // 4/0010211302021330322000 + 9264718414042152960: {3}, // 4/00102113020213303220001 + 9264718414042140672: {3}, // 4/001021130202133032200010 + 9264718414042143744: {3}, // 4/0010211302021330322000103 + 9264718414042143488: {3}, // 4/00102113020213303220001031 + 9264718414042143296: {3}, // 4/001021130202133032200010310 + 9264718414042143280: {3}, // 4/0010211302021330322000103101 + 9264718414042143284: {3}, // 4/00102113020213303220001031012 + 9264718414042143285: {3}, // 4/001021130202133032200010310122 + 9290011237591023616: {4}, // 4/0013121 + 9290064014149156864: {4}, // 4/00131213 + 9290050820009623552: {4}, // 4/001312130 + 9290047521474740224: {4}, // 4/0013121300 + 9290048346108461056: {4}, // 4/00131213003 + 9290048139950030848: {4}, // 4/001312130030 + 9290048191489638400: {4}, // 4/0013121300303 + 9290048195784605696: {4}, // 4/00131213003032 + 9290048196858347520: {4}, // 4/001312130030322 + 9290048197663653888: {4}, // 4/0013121300303223 + 9290048197864980480: {4}, // 4/00131213003032233 + 9290048197881757696: {4}, // 4/001312130030322332 + 9290048197885952000: {4}, // 4/0013121300303223322 + 9290048197882806272: {4}, // 4/00131213003032233220 + 9290048197883068416: {4}, // 4/001312130030322332202 + 9290048197883002880: {4}, // 4/0013121300303223322021 + 9290048197883052032: {4}, // 4/00131213003032233220213 + 9290048197883039744: {4}, // 4/001312130030322332202130 + 9290048197883040768: {4}, // 4/0013121300303223322021302 + 9290048197883041024: {4}, // 4/00131213003032233220213022 + 9290048197883041216: {4}, // 4/001312130030322332202130223 + 9290048197883041200: {4}, // 4/0013121300303223322021302231 + 9290048197883041212: {4}, // 4/00131213003032233220213022313 + 9290048197883041211: {4}, // 4/001312130030322332202130223131 + }, + }, + } + + for i, c := range cases { + actual := build(c.points, 6, 30) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("parse case %d \n%v, expect\n %v \n but got %v", i, c, c.expect, actual) + } + } +} diff --git a/integration/service/spatialindexer/s2indexer/dumper.go b/integration/service/spatialindexer/s2indexer/dumper.go new file mode 100644 index 00000000000..8600e0d74c6 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/dumper.go @@ -0,0 +1,146 @@ +package s2indexer + +import ( + "bytes" + "encoding/gob" + "io/ioutil" + "os" + "strings" + + "github.com/Telenav/osrm-backend/integration/pkg/api" + "github.com/golang/glog" +) + +const cellID2PointIDsFileName = "cellID2PointIDs.gob" +const pointID2LocationFileName = "pointID2Location.gob" + +func serializeS2Indexer(indexer *s2Indexer, folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath = folderPath + "/" + } + + if err := serializeCellID2PointIDs(indexer, folderPath); err != nil { + return err + } + + if err := serializePointID2Location(indexer, folderPath); err != nil { + return err + } + + glog.Infof("Successfully serialize s2Indexer to folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", + folderPath, len(indexer.cellID2PointIDs), len(indexer.pointID2Location)) + + return nil +} + +func serializeCellID2PointIDs(indexer *s2Indexer, folderPath string) error { + buf := new(bytes.Buffer) + encoder := gob.NewEncoder(buf) + err := encoder.Encode(indexer.cellID2PointIDs) + if err != nil { + glog.Errorf("During encode s2Indexer's cellID2PointIDs met error %v", err) + return err + } + if err = ioutil.WriteFile(folderPath+cellID2PointIDsFileName, buf.Bytes(), 0644); err != nil { + glog.Errorf("During dump s2Indexer's cellID2PointIDs met error %v", err) + return err + } + return nil +} + +func serializePointID2Location(indexer *s2Indexer, folderPath string) error { + buf := new(bytes.Buffer) + encoder := gob.NewEncoder(buf) + err := encoder.Encode(indexer.pointID2Location) + if err != nil { + glog.Errorf("During encode s2Indexer's pointID2Location met error %v", err) + return err + } + if err = ioutil.WriteFile(folderPath+pointID2LocationFileName, buf.Bytes(), 0644); err != nil { + glog.Errorf("During dump s2Indexer's pointID2Location met error %v", err) + return err + } + return nil +} + +func deSerializeS2Indexer(indexer *s2Indexer, folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath = folderPath + "/" + } + + if err := deSerializeCellID2PointIDs(indexer, folderPath); err != nil { + return err + } + + if err := deSerializePointID2Location(indexer, folderPath); err != nil { + return err + } + + glog.Infof("Successfully deserialize s2Indexer from folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", + folderPath, len(indexer.cellID2PointIDs), len(indexer.pointID2Location)) + + return nil +} + +func deSerializeCellID2PointIDs(indexer *s2Indexer, folderPath string) error { + byteArray, err := ioutil.ReadFile(folderPath + cellID2PointIDsFileName) + if err != nil { + glog.Errorf("During load s2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) + return err + } + buf := bytes.NewBuffer(byteArray) + decoder := gob.NewDecoder(buf) + err = decoder.Decode(&indexer.cellID2PointIDs) + if err != nil { + glog.Errorf("During decode s2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) + return err + } + + return nil +} + +func deSerializePointID2Location(indexer *s2Indexer, folderPath string) error { + byteArray, err := ioutil.ReadFile(folderPath + pointID2LocationFileName) + if err != nil { + glog.Errorf("During load s2Indexer's pointID2Location from %s met error %v", folderPath, err) + return err + } + buf := bytes.NewBuffer(byteArray) + decoder := gob.NewDecoder(buf) + err = decoder.Decode(&indexer.pointID2Location) + if err != nil { + glog.Errorf("During decode s2Indexer's pointID2Location from %s met error %v", folderPath, err) + return err + } + return nil + +} + +func removeAllDumpFiles(folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath = folderPath + "/" + } + + _, err := os.Stat(folderPath + cellID2PointIDsFileName) + if !os.IsNotExist(err) { + err = os.Remove(folderPath + cellID2PointIDsFileName) + if err != nil { + glog.Errorf("Remove file failed %s\n", folderPath+cellID2PointIDsFileName) + return err + } + } else { + glog.Warningf("There is no %s file in folder %s\n", cellID2PointIDsFileName, folderPath) + } + + _, err = os.Stat(folderPath + pointID2LocationFileName) + if !os.IsNotExist(err) { + err = os.Remove(folderPath + pointID2LocationFileName) + if err != nil { + glog.Errorf("Remove file failed %s\n", folderPath+pointID2LocationFileName) + return err + } + } else { + glog.Warningf("There is no %s file in folder %s\n", pointID2LocationFileName, folderPath) + } + return nil +} diff --git a/integration/service/spatialindexer/s2indexer/dumper_test.go b/integration/service/spatialindexer/s2indexer/dumper_test.go new file mode 100644 index 00000000000..a8393dfed05 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/dumper_test.go @@ -0,0 +1,65 @@ +package s2indexer + +import ( + "os" + "reflect" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" +) + +func TestDumper(t *testing.T) { + cases := []s2Indexer{ + s2Indexer{ + cellID2PointIDs: map[s2.CellID][]spatialindexer.PointID{ + 9263834064756932608: {1, 2, 3}, // 4/0010133 + 9263851656942977024: {1}, // 4/00101332 + 9263847258896465920: {2}, // 4/001013321 + 9263843960361582592: {3}, // 4/0010133210 + }, + pointID2Location: map[spatialindexer.PointID]spatialindexer.Location{ + 1: spatialindexer.Location{ + Latitude: 11.11, + Longitude: 11.11, + }, + 2: spatialindexer.Location{ + Latitude: 22.22, + Longitude: 22.22, + }, + 3: spatialindexer.Location{ + Latitude: 33.33, + Longitude: 33.33, + }, + }, + }, + } + // check whether curent folder is writeable + path, _ := os.Getwd() + _, err := os.OpenFile(path+"/tmp", os.O_RDONLY|os.O_CREATE, 0644) + if err != nil { + return + } + err = os.Remove(path + "/tmp") + + removeAllDumpFiles(path) + for _, c := range cases { + err := serializeS2Indexer(&c, path) + if err != nil { + t.Errorf("During running serializeS2Indexer for case %v, met error %v", c, err) + } + + actual := NewS2Indexer() + err = deSerializeS2Indexer(actual, path) + if err != nil { + t.Errorf("During running deSerializeS2Indexer for case %v, met error %v", c, err) + } + + if !reflect.DeepEqual(actual, &c) { + t.Errorf("Expect result \n%v but got \n%v\n", &c, actual) + } + + removeAllDumpFiles(path) + } + +} diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go new file mode 100644 index 00000000000..2f524529d0d --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -0,0 +1,76 @@ +package s2indexer + +import ( + "github.com/Telenav/osrm-backend/integration/service/poiloader" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" + "github.com/golang/glog" +) + +type s2Indexer struct { + cellID2PointIDs map[s2.CellID][]spatialindexer.PointID + pointID2Location map[spatialindexer.PointID]spatialindexer.Location +} + +// NewS2Indexer generates spatial indexer based on google s2 +func NewS2Indexer() *s2Indexer { + return &s2Indexer{} +} + +// Build constructs S2 indexer +func (indexer *s2Indexer) Build(filePath string) *s2Indexer { + records, err := poiloader.LoadData(filePath) + if err != nil || len(records) == 0 { + return nil + } + + var pointInfos []spatialindexer.PointInfo + for _, record := range records { + pointInfo := spatialindexer.PointInfo{ + ID: elementID2PointID(record.ID), + Location: spatialindexer.Location{ + Latitude: record.Lat, + Longitude: record.Lon, + }, + } + pointInfos = append(pointInfos, pointInfo) + + indexer.pointID2Location[elementID2PointID(record.ID)] = spatialindexer.Location{ + Latitude: record.Lat, + Longitude: record.Lon, + } + } + + indexer.cellID2PointIDs = build(pointInfos, minS2Level, maxS2Level) + return indexer +} + +// Load s2Indexer's data from contents recorded in folder +func (indexer *s2Indexer) Load(folderPath string) *s2Indexer { + if err := deSerializeS2Indexer(indexer, folderPath); err != nil { + glog.Errorf("Load s2Indexer's data from folder %s failed, err=%v\n", folderPath, err) + return nil + } + return indexer +} + +// Dump s2Indexer's content into folderPath +func (indexer *s2Indexer) Dump(folderPath string) { + if err := serializeS2Indexer(indexer, folderPath); err != nil { + glog.Errorf("Dump s2Indexer's data to folder %s failed, err=%v\n", folderPath, err) + } +} + +func (indexer s2Indexer) getPointLocationByPointID(id spatialindexer.PointID) (spatialindexer.Location, bool) { + location, ok := indexer.pointID2Location[id] + return location, ok +} + +func (indexer s2Indexer) getPointIDsByS2CellID(cellid s2.CellID) ([]spatialindexer.PointID, bool) { + pointIDs, ok := indexer.cellID2PointIDs[cellid] + return pointIDs, ok +} + +func elementID2PointID(id int64) spatialindexer.PointID { + return (spatialindexer.PointID)(id) +} diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go new file mode 100644 index 00000000000..0024a4fd4f0 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -0,0 +1,76 @@ +package s2indexer + +import ( + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s1" + "github.com/golang/geo/s2" + "github.com/golang/glog" +) + +const maxCellCount = 200 + +const s2EarthRadiusInMeters = 6371010.0 + +func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) []s2.CellID { + regionCover := &s2.RegionCoverer{ + MinLevel: minS2Level, + MaxLevel: 20, + MaxCells: maxCellCount} + center := s2.PointFromLatLng(s2.LatLngFromDegrees(point.Latitude, point.Longitude)) + radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) + region := s2.Region(s2.CapFromCenterAngle(center, radius)) + cellUnion := regionCover.Covering(region) + generateDebugInfo4CellIDs(([]s2.CellID)(cellUnion)) + return ([]s2.CellID)(cellUnion) +} + +func queryNearByPoints(indexer *s2Indexer, point spatialindexer.Location, radius float64) []spatialindexer.PointInfo { + var result []spatialindexer.PointInfo + cellIDs := queryNearByS2Cells(point, radius) + for _, cellID := range cellIDs { + pointIDs, hasCellID := indexer.getPointIDsByS2CellID(cellID) + if !hasCellID { + continue + } + + for _, pointID := range pointIDs { + location, hasPointID := indexer.getPointLocationByPointID(pointID) + if !hasPointID { + glog.Errorf("In queryNearByPoints, use incorrect pointID %v to query s2Indexer\n", pointID) + continue + } + + result = append(result, spatialindexer.PointInfo{ + ID: pointID, + Location: location, + }) + } + } + + return result +} + +func generateDebugInfo4CellIDs(cellIDs []s2.CellID) { + glog.Info("=================================\n") + glog.Info("generateDebugInfo4CellIDs\n") + for _, cellID := range cellIDs { + glog.Infof("CellID value = %d(uint64), string = %v, token = %s, level = %d\n", + (uint64)(cellID), cellID, cellID.ToToken(), cellID.Level()) + } + glog.Info("=================================\n") +} + +// Generate sample url like: http://s2.sidewalklabs.com/regioncoverer/?cells=89c2584b54,89c2584d,89c25852c,89c259a5,89c259ac,89c259b4,89c259bc,89c259c7,89c259c9,89c259ca4 +func generateDebugURL(cellIDs []s2.CellID) string { + var url string + + if len(cellIDs) == 0 { + return url + } + + url += "http://s2.sidewalklabs.com/regioncoverer/?cells=" + for _, cellID := range cellIDs { + url += cellID.ToToken() + "," + } + return url +} diff --git a/integration/service/spatialindexer/s2indexer/spatial_query_test.go b/integration/service/spatialindexer/s2indexer/spatial_query_test.go new file mode 100644 index 00000000000..2b9599610a0 --- /dev/null +++ b/integration/service/spatialindexer/s2indexer/spatial_query_test.go @@ -0,0 +1,245 @@ +package s2indexer + +import ( + "reflect" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/geo/s2" + "github.com/golang/glog" +) + +func TestSpatialIndexQuery1(t *testing.T) { + fakeIndexer1 := s2Indexer{ + cellID2PointIDs: map[s2.CellID][]spatialindexer.PointID{ + 9263834064756932608: {1, 2}, // 4/0010133 + 9263851656942977024: {1, 2}, // 4/00101332 + 9263847258896465920: {1, 2}, // 4/001013321 + 9263843960361582592: {1, 2}, // 4/0010133210 + 9263844784995303424: {1, 2}, // 4/00101332103 + 9263844716275826688: {2}, // 4/001013321031 + 9263844733455695872: {2}, // 4/0010133210312 + 9263844746340597760: {2}, // 4/00101332103123 + 9263844749561823232: {2}, // 4/001013321031233 + 9263844749830258688: {2}, // 4/0010133210312332 + 9263844750031585280: {2}, // 4/00101332103123323 + 9263844749981253632: {2}, // 4/001013321031233230 + 9263844749993836544: {2}, // 4/0010133210312332303 + 9263844749996982272: {2}, // 4/00101332103123323033 + 9263844749996720128: {2}, // 4/001013321031233230331 + 9263844749996916736: {2}, // 4/0010133210312332303313 + 9263844749996900352: {2}, // 4/00101332103123323033131 + 9263844749996896256: {2}, // 4/001013321031233230331311 + 9263844749996893184: {2}, // 4/0010133210312332303313110 + 9263844749996892416: {2}, // 4/00101332103123323033131100 + 9263844749996892608: {2}, // 4/001013321031233230331311003 + 9263844749996892592: {2}, // 4/0010133210312332303313110031 + 9263844749996892588: {2}, // 4/00101332103123323033131100311 + 9263844749996892591: {2}, // 4/001013321031233230331311003113 + 9263844853714780160: {1}, // 4/001013321032 + 9263844836534910976: {1}, // 4/0010133210321 + 9263844823650009088: {1}, // 4/00101332103210 + 9263844822576267264: {1}, // 4/001013321032101 + 9263844823381573632: {1}, // 4/0010133210321013 + 9263844823180247040: {1}, // 4/00101332103210130 + 9263844823129915392: {1}, // 4/001013321032101300 + 9263844823134109696: {1}, // 4/0010133210321013002 + 9263844823130963968: {1}, // 4/00101332103210130020 + 9263844823130701824: {1}, // 4/001013321032101300201 + 9263844823130898432: {1}, // 4/0010133210321013002013 + 9263844823130947584: {1}, // 4/00101332103210130020133 + 9263844823130943488: {1}, // 4/001013321032101300201331 + 9263844823130944512: {1}, // 4/0010133210321013002013312 + 9263844823130943744: {1}, // 4/00101332103210130020133120 + 9263844823130943680: {1}, // 4/001013321032101300201331201 + 9263844823130943696: {1}, // 4/0010133210321013002013312012 + 9263844823130943708: {1}, // 4/00101332103210130020133120123 + 9263844823130943709: {1}, // 4/001013321032101300201331201232 + }, + pointID2Location: map[spatialindexer.PointID]spatialindexer.Location{ + 1: spatialindexer.Location{ + Latitude: 37.402701, + Longitude: -121.974096, + }, + 2: spatialindexer.Location{ + Latitude: 37.403530, + Longitude: -121.969768, + }, + }, + } + + // center in 4655 great america pkwy + center := spatialindexer.Location{ + Latitude: 37.402799, + Longitude: -121.969861, + } + + expect := []spatialindexer.PointInfo{ + spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Latitude: 37.402701, + Longitude: -121.974096, + }, + }, + spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Latitude: 37.40353, + Longitude: -121.969768, + }, + }, + } + + actual := queryNearByPoints(&fakeIndexer1, center, 10000) + + if !reflect.DeepEqual(actual, expect) { + t.Errorf("Expect result is \n%v but got \n%v\n", actual, expect) + } +} + +//More information could go to here: https://github.com/Telenav/osrm-backend/issues/236#issuecomment-603533484 +func TestQueryNearByS2Cells1(t *testing.T) { + // center in 4655 great america pkwy + center := spatialindexer.Location{ + Latitude: 37.402799, + Longitude: -121.969861, + } + actualCellIDs := queryNearByS2Cells(center, 1600) + glog.Infof("\nTest URL is %s\n", generateDebugURL(actualCellIDs)) + + expectCellIDs := []s2.CellID{ + 9263843046942834688, // token = 808fc82b54 + 9263843047026720768, // token = 808fc82b59 + 9263843047131578368, // token = 808fc82b5f4 + 9263843050566713344, // token = 808fc82c2c + 9263843050700931072, // token = 808fc82c34 + 9263843050835148800, // token = 808fc82c3c + 9263843051170693120, // token = 808fc82c5 + 9263843051506237440, // token = 808fc82c64 + 9263843051590123520, // token = 808fc82c69 + 9263843053049741312, // token = 808fc82cc + 9263843055197224960, // token = 808fc82d4 + 9263843057344708608, // token = 808fc82dc + 9263843058686885888, // token = 808fc82e1 + 9263843060364607488, // token = 808fc82e74 + 9263843060448493568, // token = 808fc82e79 + 9263843072377094144, // token = 808fc8314 + 9263843073719271424, // token = 808fc8319 + 9263843074256142336, // token = 808fc831b + 9263843074591686656, // token = 808fc831c4 + 9263843074994339840, // token = 808fc831dc + 9263843075329884160, // token = 808fc831f + 9263843079893286912, // token = 808fc833 + 9263843088483221504, // token = 808fc835 + 9263843093046624256, // token = 808fc8361 + 9263843093365391360, // token = 808fc83623 + 9263843093784821760, // token = 808fc8363c + 9263843093919039488, // token = 808fc83644 + 9263843094321692672, // token = 808fc8365c + 9263843094657236992, // token = 808fc8367 + 9263843095999414272, // token = 808fc836c + 9263843176525856768, // token = 808fc8497fc + 9263843176798486528, // token = 808fc8499 + 9263843178409099264, // token = 808fc849f + 9263843182972502016, // token = 808fc84b + 9263843188810973184, // token = 808fc84c5c + 9263843189146517504, // token = 808fc84c7 + 9263843190488694784, // token = 808fc84cc + 9263843192032198656, // token = 808fc84d1c + 9263843192367742976, // token = 808fc84d3 + 9263843192904613888, // token = 808fc84d5 + 9263844597090484224, // token = 808fc9944 + 9263844598432661504, // token = 808fc9949 + 9263844599036641280, // token = 808fc994b4 + 9263844599170859008, // token = 808fc994bc + 9263844599506403328, // token = 808fc994d + 9263844600043274240, // token = 808fc994f + 9263844601385451520, // token = 808fc9954 + 9263844603532935168, // token = 808fc995c + 9263844608901644288, // token = 808fc997 + 9263844614270353408, // token = 808fc9984 + 9263844615612530688, // token = 808fc9989 + 9263844616149401600, // token = 808fc998b + 9263844616484945920, // token = 808fc998c4 + 9263844617223143424, // token = 808fc998f + 9263844617558687744, // token = 808fc99904 + 9263844620444368896, // token = 808fc999b + 9263844620981239808, // token = 808fc999d + 9263844621316784128, // token = 808fc999e4 + 9263844621451001856, // token = 808fc999ec + 9263844653193494528, // token = 808fc9a15 + 9263844653730365440, // token = 808fc9a17 + 9263844654267236352, // token = 808fc9a19 + 9263844655944957952, // token = 808fc9a1f4 + 9263844656028844032, // token = 808fc9a1f9 + 9263844660441251840, // token = 808fc9a3 + 9263844669031186432, // token = 808fc9a5 + 9263844673594589184, // token = 808fc9a61 + 9263844675205201920, // token = 808fc9a67 + 9263844675742072832, // token = 808fc9a69 + 9263844676278943744, // token = 808fc9a6b + 9263844684801769472, // token = 808fc9a8ac + 9263844684935987200, // token = 808fc9a8b4 + 9263844685316620288, // token = 808fc9a8cab + 9263844699901263872, // token = 808fc9ac3 + 9263844703592251392, // token = 808fc9ad0c + 9263844704934428672, // token = 808fc9ad5c + 9263844705269972992, // token = 808fc9ad7 + 9263844706612150272, // token = 808fc9adc + 9263844708759633920, // token = 808fc9ae4 + 9263844710101811200, // token = 808fc9ae9 + 9263844710840008704, // token = 808fc9aebc + 9263844711175553024, // token = 808fc9aed + 9263844711712423936, // token = 808fc9aef + 9263844713054601216, // token = 808fc9af4 + 9263844715202084864, // token = 808fc9afc + 9263844733455695872, // token = 808fc9b4 + 9263844767815434240, // token = 808fc9bc + 9263844789290270720, // token = 808fc9c1 + 9263844794658979840, // token = 808fc9c24 + 9263844795934048256, // token = 808fc9c28c + 9263844796851552256, // token = 808fc9c2c2b + 9263844797276225536, // token = 808fc9c2dc + 9263844797611769856, // token = 808fc9c2f + 9263844798953947136, // token = 808fc9c34 + 9263844801101430784, // token = 808fc9c3c + 9263844806470139904, // token = 808fc9c5 + 9263844815060074496, // token = 808fc9c7 + 9263844836534910976, // token = 808fc9cc + 9263844858009747456, // token = 808fc9d1 + 9263844866599682048, // token = 808fc9d3 + 9263844871163084800, // token = 808fc9d41 + 9263844872773697536, // token = 808fc9d47 + 9263844873310568448, // token = 808fc9d49 + 9263844873646112768, // token = 808fc9d4a4 + 9263844873780330496, // token = 808fc9d4ac + 9263844882437373952, // token = 808fc9d6b + 9263844882907136000, // token = 808fc9d6cc + 9263844883041353728, // token = 808fc9d6d4 + 9263844884517748736, // token = 808fc9d72c + 9263844896865779712, // token = 808fc9da0c + 9263844896999997440, // token = 808fc9da14 + 9263844898006630400, // token = 808fc9da5 + 9263844898543501312, // token = 808fc9da7 + 9263844899885678592, // token = 808fc9dac + 9263844902033162240, // token = 808fc9db4 + 9263844903710883840, // token = 808fc9dba4 + 9263844903845101568, // token = 808fc9dbac + 9263844907921965056, // token = 808fc9dc9f + 9263844908207177728, // token = 808fc9dcb + 9263844908676939776, // token = 808fc9dccc + 9263844964779950080, // token = 808fc9e9dc + 9263844965115494400, // token = 808fc9e9f + 9263844966457671680, // token = 808fc9ea4 + 9263844968605155328, // token = 808fc9eac + 9263844969947332608, // token = 808fc9eb1 + 9263844970484203520, // token = 808fc9eb3 + 9263844970937188352, // token = 808fc9eb4b + 9263844972564578304, // token = 808fc9ebac + } + + if !reflect.DeepEqual(actualCellIDs, expectCellIDs) { + t.Errorf("Expect result is \n%v but got \n%v\n", expectCellIDs, actualCellIDs) + } +} From 8f9e8684ea8686f7535a400612d2b5b8dd113abc Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 16:57:18 -0700 Subject: [PATCH 02/27] fix: remove hard code value. issue: https://github.com/Telenav/osrm-backend/issues/239 --- integration/service/spatialindexer/s2indexer/spatial_query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go index 0024a4fd4f0..3c2d903cfd9 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -14,7 +14,7 @@ const s2EarthRadiusInMeters = 6371010.0 func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) []s2.CellID { regionCover := &s2.RegionCoverer{ MinLevel: minS2Level, - MaxLevel: 20, + MaxLevel: maxS2Level, MaxCells: maxCellCount} center := s2.PointFromLatLng(s2.LatLngFromDegrees(point.Latitude, point.Longitude)) radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) From 459b170aafec0b51d2f748faf72e1057ecef0a8d Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 18:01:24 -0700 Subject: [PATCH 03/27] fix: refactor code. --- integration/service/spatialindexer/doc.go | 9 +++++---- integration/service/spatialindexer/iterface.go | 17 ++++++++++++++--- .../service/spatialindexer/s2indexer/builder.go | 6 +++--- .../spatialindexer/s2indexer/builder_test.go | 2 +- .../service/spatialindexer/s2indexer/dumper.go | 1 + .../spatialindexer/s2indexer/spatial_query.go | 1 - 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/integration/service/spatialindexer/doc.go b/integration/service/spatialindexer/doc.go index 2e98a65aaeb..3e848cc3a20 100644 --- a/integration/service/spatialindexer/doc.go +++ b/integration/service/spatialindexer/doc.go @@ -1,6 +1,7 @@ -// package spatialindexer answers query of nearest pois for given point +// package spatialindexer answers query of nearest points(place, point of interest) for conditions +// such as center location, radius, etc // -// Sample 1: Build connectivity for charge stations during pro-processing +// Sample Scenario 1: Build connectivity for charge stations during pro-processing // indexer := NewS2Indexer().Build(poiCsvFile) // for _, stationPoint := range chargeStations { // nearbyStations := indexer.FindNearByIDs(stationPoint, 800km, -1) @@ -8,11 +9,11 @@ // } // // -// Sample 2: Dump s2Indexer's content to folder +// Sample Scenario 2: Dump s2Indexer's content to folder // indexer.Dump(folderPath) // // -// Sample 3: Query reachable charge stations with current energy level +// Sample Scenario 3: Query reachable charge stations with current energy level // indexer := NewS2Indexer().Load(folderPath) // nearbyStations := indexer.FindNearByIDs(currentPoint, currentEnergyLevel, -1) // diff --git a/integration/service/spatialindexer/iterface.go b/integration/service/spatialindexer/iterface.go index e8213635271..dfcca077f7c 100644 --- a/integration/service/spatialindexer/iterface.go +++ b/integration/service/spatialindexer/iterface.go @@ -7,24 +7,35 @@ type Location struct { Longitude float64 } +// PointInfo records point related information such as ID and location type PointInfo struct { ID PointID Location Location } +// RankedPointInfo used to record ranking result, distance to specific point could be used for ranking type RankedPointInfo struct { PointInfo Distance float64 } +// PointID defines ID for given point(location, point of interest) +// Only the data used for pre-processing contains valid PointID type PointID int64 +// Finder answers special query type Finder interface { - FindNearByIDs(center Location, radius float64, limitCount int) []PointInfo + + // FindNearByIDs returns a group of points near to given center location + FindNearByPointIDs(center Location, radius float64, limitCount int) []PointInfo } +// Ranker used to ranking a group of points type Ranker interface { - RankingIDsByGreatCircleDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo - RankingIDsByShortestDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo + // RankPointIDsByGreatCircleDistance ranks a group of points based on great circle distance to given location + RankPointIDsByGreatCircleDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo + + // RankPointIDsByShortestDistance ranks a group of points based on shortest path distance to given location + RankPointIDsByShortestDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo } diff --git a/integration/service/spatialindexer/s2indexer/builder.go b/integration/service/spatialindexer/s2indexer/builder.go index c7ef012d7bf..eed79895dbb 100644 --- a/integration/service/spatialindexer/s2indexer/builder.go +++ b/integration/service/spatialindexer/s2indexer/builder.go @@ -14,16 +14,16 @@ const minS2Level = 9 // Level = 20 means average area size is 77.32km2 const maxS2Level = 20 -func build(pois []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.CellID][]spatialindexer.PointID { +func build(points []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.CellID][]spatialindexer.PointID { pointID2CellIDs := make(map[spatialindexer.PointID][]s2.CellID) cellID2PointIDs := make(map[s2.CellID][]spatialindexer.PointID) - for _, p := range pois { + for _, p := range points { leafCellID := s2.CellFromLatLng(s2.LatLngFromDegrees(p.Location.Latitude, p.Location.Longitude)).ID() var cellIDs []s2.CellID // For level = 30, its parent equal to current - // So no need append leafCellID outside of for loop if maxLevel = 30 + // So no need append leafCellID into cellIDs outside of for loop for i := leafCellID.Level(); i > minLevel; i-- { if i > maxLevel { continue diff --git a/integration/service/spatialindexer/s2indexer/builder_test.go b/integration/service/spatialindexer/s2indexer/builder_test.go index 4fa3c8e3847..7a54a8862d9 100644 --- a/integration/service/spatialindexer/s2indexer/builder_test.go +++ b/integration/service/spatialindexer/s2indexer/builder_test.go @@ -8,7 +8,7 @@ import ( "github.com/golang/geo/s2" ) -// online tool for google s2: http://s2map.com/#order=latlng&mode=polygon&s2=false&points= +// online tool for google s2: http://s2.sidewalklabs.com/regioncoverer/ func TestBuild(t *testing.T) { cases := []struct { points []spatialindexer.PointInfo diff --git a/integration/service/spatialindexer/s2indexer/dumper.go b/integration/service/spatialindexer/s2indexer/dumper.go index 8600e0d74c6..d70fa978785 100644 --- a/integration/service/spatialindexer/s2indexer/dumper.go +++ b/integration/service/spatialindexer/s2indexer/dumper.go @@ -142,5 +142,6 @@ func removeAllDumpFiles(folderPath string) error { } else { glog.Warningf("There is no %s file in folder %s\n", pointID2LocationFileName, folderPath) } + return nil } diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go index 3c2d903cfd9..0cd26b65d75 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -20,7 +20,6 @@ func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) [ radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) region := s2.Region(s2.CapFromCenterAngle(center, radius)) cellUnion := regionCover.Covering(region) - generateDebugInfo4CellIDs(([]s2.CellID)(cellUnion)) return ([]s2.CellID)(cellUnion) } From cb08f933845b169010afea0e3087334a9a2e3fb5 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 18:28:33 -0700 Subject: [PATCH 04/27] fix: remove local changes. --- integration/go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/go.sum b/integration/go.sum index 6c3a3f88da6..4eaede4a194 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1,6 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Telenav/osrm-backend v5.22.0+incompatible h1:v5K4VmdxyQjFXN20ZGmk9MchGTHWWIMUXiXmAndNvlQ= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/golang/geo v0.0.0-20200319012246-673a6f80352d h1:C/hKUcHT483btRbeGkrRjJz+Zbcj8audldIi9tRJDCc= github.com/golang/geo v0.0.0-20200319012246-673a6f80352d/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= From 6de90a82c465629c00d318c2e8b8bd5261c3b52f Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 18:32:55 -0700 Subject: [PATCH 05/27] fix: update comments --- integration/service/poiloader/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/service/poiloader/loader.go b/integration/service/poiloader/loader.go index 5f86e98bfa1..7190c30a2fe 100644 --- a/integration/service/poiloader/loader.go +++ b/integration/service/poiloader/loader.go @@ -7,7 +7,7 @@ import ( "github.com/golang/glog" ) -// LoadPOI accepts json file recorded with poi data and returns deserialized result +// LoadData accepts json file with points data and returns deserialized result func LoadData(filePath string) ([]Element, error) { var elements []Element From 4dcf8f0c836b67c8001aec77de0cdae1fc5a980f Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 19:55:37 -0700 Subject: [PATCH 06/27] fix: add fake main to pass ci. --- integration/cmd/chargestations-connectivity-gen/main.go | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 integration/cmd/chargestations-connectivity-gen/main.go diff --git a/integration/cmd/chargestations-connectivity-gen/main.go b/integration/cmd/chargestations-connectivity-gen/main.go new file mode 100644 index 00000000000..934fe5d5111 --- /dev/null +++ b/integration/cmd/chargestations-connectivity-gen/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "fmt" +) + +func main() { + fmt.Print("Hello World!") +} From 240bb5b0a90080d6686191c2ff09c2d5a9160d3b Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 24 Mar 2020 20:31:14 -0700 Subject: [PATCH 07/27] fix: fix issue in unit test. Build()'s result no need to guarantee the sequence --- .../service/spatialindexer/s2indexer/builder_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/integration/service/spatialindexer/s2indexer/builder_test.go b/integration/service/spatialindexer/s2indexer/builder_test.go index 7a54a8862d9..92b5c1a7502 100644 --- a/integration/service/spatialindexer/s2indexer/builder_test.go +++ b/integration/service/spatialindexer/s2indexer/builder_test.go @@ -2,6 +2,7 @@ package s2indexer import ( "reflect" + "sort" "testing" "github.com/Telenav/osrm-backend/integration/service/spatialindexer" @@ -253,6 +254,14 @@ func TestBuild(t *testing.T) { for i, c := range cases { actual := build(c.points, 6, 30) + + // sort slice to directly use DeepEqual, Build()'s result no need to guarantee the sequence + for k := range actual { + sort.Slice(actual[k], func(i, j int) bool { + return actual[k][i] < actual[k][j] + }) + } + if !reflect.DeepEqual(actual, c.expect) { t.Errorf("parse case %d \n%v, expect\n %v \n but got %v", i, c, c.expect, actual) } From 82705722b2c32a9dee5445a8957f5751cd98a4d4 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Thu, 26 Mar 2020 11:11:11 -0700 Subject: [PATCH 08/27] fix: adjust code based on review. --- .../chargestations-connectivity-gen/main.go | 1 + integration/service/spatialindexer/doc.go | 2 +- .../{iterface.go => interface.go} | 2 +- .../{ => spatialindexer}/poiloader/loader.go | 0 .../poiloader/poi_format.go | 6 +++ .../poiloader/sample_input.json | 0 .../spatialindexer/s2indexer/builder.go | 3 ++ .../spatialindexer/s2indexer/dumper.go | 50 +++++++++++-------- .../spatialindexer/s2indexer/dumper_test.go | 4 +- .../spatialindexer/s2indexer/indexer.go | 28 ++++++----- .../spatialindexer/s2indexer/spatial_query.go | 10 +++- .../s2indexer/spatial_query_test.go | 2 +- 12 files changed, 68 insertions(+), 40 deletions(-) rename integration/service/spatialindexer/{iterface.go => interface.go} (93%) rename integration/service/{ => spatialindexer}/poiloader/loader.go (100%) rename integration/service/{ => spatialindexer}/poiloader/poi_format.go (84%) rename integration/service/{ => spatialindexer}/poiloader/sample_input.json (100%) diff --git a/integration/cmd/chargestations-connectivity-gen/main.go b/integration/cmd/chargestations-connectivity-gen/main.go index 934fe5d5111..bd1acb5d084 100644 --- a/integration/cmd/chargestations-connectivity-gen/main.go +++ b/integration/cmd/chargestations-connectivity-gen/main.go @@ -5,5 +5,6 @@ import ( ) func main() { + // @todo: add logic to generate connectivity for charge stations fmt.Print("Hello World!") } diff --git a/integration/service/spatialindexer/doc.go b/integration/service/spatialindexer/doc.go index 3e848cc3a20..87b49a73af4 100644 --- a/integration/service/spatialindexer/doc.go +++ b/integration/service/spatialindexer/doc.go @@ -9,7 +9,7 @@ // } // // -// Sample Scenario 2: Dump s2Indexer's content to folder +// Sample Scenario 2: Dump S2Indexer's content to folder // indexer.Dump(folderPath) // // diff --git a/integration/service/spatialindexer/iterface.go b/integration/service/spatialindexer/interface.go similarity index 93% rename from integration/service/spatialindexer/iterface.go rename to integration/service/spatialindexer/interface.go index dfcca077f7c..7d0aaf5138c 100644 --- a/integration/service/spatialindexer/iterface.go +++ b/integration/service/spatialindexer/interface.go @@ -26,7 +26,7 @@ type PointID int64 // Finder answers special query type Finder interface { - // FindNearByIDs returns a group of points near to given center location + // FindNearByPointIDs returns a group of points near to given center location FindNearByPointIDs(center Location, radius float64, limitCount int) []PointInfo } diff --git a/integration/service/poiloader/loader.go b/integration/service/spatialindexer/poiloader/loader.go similarity index 100% rename from integration/service/poiloader/loader.go rename to integration/service/spatialindexer/poiloader/loader.go diff --git a/integration/service/poiloader/poi_format.go b/integration/service/spatialindexer/poiloader/poi_format.go similarity index 84% rename from integration/service/poiloader/poi_format.go rename to integration/service/spatialindexer/poiloader/poi_format.go index 95c3b5e4055..746a7456555 100644 --- a/integration/service/poiloader/poi_format.go +++ b/integration/service/spatialindexer/poiloader/poi_format.go @@ -1,5 +1,6 @@ package poiloader +// Element represent information loaded for point record type Element struct { ID int64 `json:"id"` VendorCode VendorCode `json:"vendor_code"` @@ -32,24 +33,28 @@ type Element struct { Amenity Amenity `json:"amenity"` } +// Amenity is defined by OSM format: https://wiki.openstreetmap.org/wiki/Tag:amenity%3Dcharging_station type Amenity string const ( ChargingStation Amenity = "charging_station" ) +// Country records IOS code type Country string const ( Usa Country = "USA" ) +// SpaceID is used to define a code for political defined administrative area type SpaceID string const ( UsaCA SpaceID = "USA_CA" ) +// VendorCode defines data source provider type VendorCode string const ( @@ -58,6 +63,7 @@ const ( Nolp VendorCode = "NOLP" ) +// CategoryIDGather records aggregate data for certain type of point type CategoryIDGather struct { Integer *int64 String *string diff --git a/integration/service/poiloader/sample_input.json b/integration/service/spatialindexer/poiloader/sample_input.json similarity index 100% rename from integration/service/poiloader/sample_input.json rename to integration/service/spatialindexer/poiloader/sample_input.json diff --git a/integration/service/spatialindexer/s2indexer/builder.go b/integration/service/spatialindexer/s2indexer/builder.go index eed79895dbb..49550b7ef47 100644 --- a/integration/service/spatialindexer/s2indexer/builder.go +++ b/integration/service/spatialindexer/s2indexer/builder.go @@ -28,9 +28,11 @@ func build(points []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.Cel if i > maxLevel { continue } + parentCellID := leafCellID.Parent(i) cellIDs = append(cellIDs, parentCellID) } + pointID2CellIDs[p.ID] = cellIDs } @@ -40,6 +42,7 @@ func build(points []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.Cel var pointIDs []spatialindexer.PointID cellID2PointIDs[cellID] = pointIDs } + cellID2PointIDs[cellID] = append(cellID2PointIDs[cellID], pointID) } } diff --git a/integration/service/spatialindexer/s2indexer/dumper.go b/integration/service/spatialindexer/s2indexer/dumper.go index d70fa978785..0432a6d5a4b 100644 --- a/integration/service/spatialindexer/s2indexer/dumper.go +++ b/integration/service/spatialindexer/s2indexer/dumper.go @@ -14,9 +14,9 @@ import ( const cellID2PointIDsFileName = "cellID2PointIDs.gob" const pointID2LocationFileName = "pointID2Location.gob" -func serializeS2Indexer(indexer *s2Indexer, folderPath string) error { +func serializeS2Indexer(indexer *S2Indexer, folderPath string) error { if !strings.HasSuffix(folderPath, api.Slash) { - folderPath = folderPath + "/" + folderPath += "/" } if err := serializeCellID2PointIDs(indexer, folderPath); err != nil { @@ -27,45 +27,51 @@ func serializeS2Indexer(indexer *s2Indexer, folderPath string) error { return err } - glog.Infof("Successfully serialize s2Indexer to folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", + glog.Infof("Successfully serialize S2Indexer to folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", folderPath, len(indexer.cellID2PointIDs), len(indexer.pointID2Location)) return nil } -func serializeCellID2PointIDs(indexer *s2Indexer, folderPath string) error { +func serializeCellID2PointIDs(indexer *S2Indexer, folderPath string) error { buf := new(bytes.Buffer) encoder := gob.NewEncoder(buf) err := encoder.Encode(indexer.cellID2PointIDs) + if err != nil { - glog.Errorf("During encode s2Indexer's cellID2PointIDs met error %v", err) + glog.Errorf("During encode S2Indexer's cellID2PointIDs met error %v", err) return err } + if err = ioutil.WriteFile(folderPath+cellID2PointIDsFileName, buf.Bytes(), 0644); err != nil { - glog.Errorf("During dump s2Indexer's cellID2PointIDs met error %v", err) + glog.Errorf("During dump S2Indexer's cellID2PointIDs met error %v", err) return err } + return nil } -func serializePointID2Location(indexer *s2Indexer, folderPath string) error { +func serializePointID2Location(indexer *S2Indexer, folderPath string) error { buf := new(bytes.Buffer) encoder := gob.NewEncoder(buf) err := encoder.Encode(indexer.pointID2Location) + if err != nil { - glog.Errorf("During encode s2Indexer's pointID2Location met error %v", err) + glog.Errorf("During encode S2Indexer's pointID2Location met error %v", err) return err } + if err = ioutil.WriteFile(folderPath+pointID2LocationFileName, buf.Bytes(), 0644); err != nil { - glog.Errorf("During dump s2Indexer's pointID2Location met error %v", err) + glog.Errorf("During dump S2Indexer's pointID2Location met error %v", err) return err } + return nil } -func deSerializeS2Indexer(indexer *s2Indexer, folderPath string) error { +func deSerializeS2Indexer(indexer *S2Indexer, folderPath string) error { if !strings.HasSuffix(folderPath, api.Slash) { - folderPath = folderPath + "/" + folderPath += "/" } if err := deSerializeCellID2PointIDs(indexer, folderPath); err != nil { @@ -76,49 +82,53 @@ func deSerializeS2Indexer(indexer *s2Indexer, folderPath string) error { return err } - glog.Infof("Successfully deserialize s2Indexer from folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", + glog.Infof("Successfully deserialize S2Indexer from folder %s. len(indexer.cellID2PointIDs) = %d, len(indexer.pointID2Location) = %d\n", folderPath, len(indexer.cellID2PointIDs), len(indexer.pointID2Location)) return nil } -func deSerializeCellID2PointIDs(indexer *s2Indexer, folderPath string) error { +func deSerializeCellID2PointIDs(indexer *S2Indexer, folderPath string) error { byteArray, err := ioutil.ReadFile(folderPath + cellID2PointIDsFileName) if err != nil { - glog.Errorf("During load s2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) + glog.Errorf("During load S2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) return err } + buf := bytes.NewBuffer(byteArray) decoder := gob.NewDecoder(buf) err = decoder.Decode(&indexer.cellID2PointIDs) + if err != nil { - glog.Errorf("During decode s2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) + glog.Errorf("During decode S2Indexer's cellID2PointIDs from %s met error %v", folderPath, err) return err } return nil } -func deSerializePointID2Location(indexer *s2Indexer, folderPath string) error { +func deSerializePointID2Location(indexer *S2Indexer, folderPath string) error { byteArray, err := ioutil.ReadFile(folderPath + pointID2LocationFileName) if err != nil { - glog.Errorf("During load s2Indexer's pointID2Location from %s met error %v", folderPath, err) + glog.Errorf("During load S2Indexer's pointID2Location from %s met error %v", folderPath, err) return err } + buf := bytes.NewBuffer(byteArray) decoder := gob.NewDecoder(buf) err = decoder.Decode(&indexer.pointID2Location) + if err != nil { - glog.Errorf("During decode s2Indexer's pointID2Location from %s met error %v", folderPath, err) + glog.Errorf("During decode S2Indexer's pointID2Location from %s met error %v", folderPath, err) return err } - return nil + return nil } func removeAllDumpFiles(folderPath string) error { if !strings.HasSuffix(folderPath, api.Slash) { - folderPath = folderPath + "/" + folderPath += "/" } _, err := os.Stat(folderPath + cellID2PointIDsFileName) diff --git a/integration/service/spatialindexer/s2indexer/dumper_test.go b/integration/service/spatialindexer/s2indexer/dumper_test.go index a8393dfed05..5fd860dae76 100644 --- a/integration/service/spatialindexer/s2indexer/dumper_test.go +++ b/integration/service/spatialindexer/s2indexer/dumper_test.go @@ -10,8 +10,8 @@ import ( ) func TestDumper(t *testing.T) { - cases := []s2Indexer{ - s2Indexer{ + cases := []S2Indexer{ + S2Indexer{ cellID2PointIDs: map[s2.CellID][]spatialindexer.PointID{ 9263834064756932608: {1, 2, 3}, // 4/0010133 9263851656942977024: {1}, // 4/00101332 diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go index 2f524529d0d..6806555078c 100644 --- a/integration/service/spatialindexer/s2indexer/indexer.go +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -1,30 +1,32 @@ package s2indexer import ( - "github.com/Telenav/osrm-backend/integration/service/poiloader" "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer/poiloader" "github.com/golang/geo/s2" "github.com/golang/glog" ) -type s2Indexer struct { +// S2Indexer provide spatial index ability based on google s2 +type S2Indexer struct { cellID2PointIDs map[s2.CellID][]spatialindexer.PointID pointID2Location map[spatialindexer.PointID]spatialindexer.Location } // NewS2Indexer generates spatial indexer based on google s2 -func NewS2Indexer() *s2Indexer { - return &s2Indexer{} +func NewS2Indexer() *S2Indexer { + return &S2Indexer{} } // Build constructs S2 indexer -func (indexer *s2Indexer) Build(filePath string) *s2Indexer { +func (indexer *S2Indexer) Build(filePath string) *S2Indexer { records, err := poiloader.LoadData(filePath) if err != nil || len(records) == 0 { return nil } var pointInfos []spatialindexer.PointInfo + for _, record := range records { pointInfo := spatialindexer.PointInfo{ ID: elementID2PointID(record.ID), @@ -45,28 +47,28 @@ func (indexer *s2Indexer) Build(filePath string) *s2Indexer { return indexer } -// Load s2Indexer's data from contents recorded in folder -func (indexer *s2Indexer) Load(folderPath string) *s2Indexer { +// Load S2Indexer's data from contents recorded in folder +func (indexer *S2Indexer) Load(folderPath string) *S2Indexer { if err := deSerializeS2Indexer(indexer, folderPath); err != nil { - glog.Errorf("Load s2Indexer's data from folder %s failed, err=%v\n", folderPath, err) + glog.Errorf("Load S2Indexer's data from folder %s failed, err=%v\n", folderPath, err) return nil } return indexer } -// Dump s2Indexer's content into folderPath -func (indexer *s2Indexer) Dump(folderPath string) { +// Dump S2Indexer's content into folderPath +func (indexer *S2Indexer) Dump(folderPath string) { if err := serializeS2Indexer(indexer, folderPath); err != nil { - glog.Errorf("Dump s2Indexer's data to folder %s failed, err=%v\n", folderPath, err) + glog.Errorf("Dump S2Indexer's data to folder %s failed, err=%v\n", folderPath, err) } } -func (indexer s2Indexer) getPointLocationByPointID(id spatialindexer.PointID) (spatialindexer.Location, bool) { +func (indexer S2Indexer) getPointLocationByPointID(id spatialindexer.PointID) (spatialindexer.Location, bool) { location, ok := indexer.pointID2Location[id] return location, ok } -func (indexer s2Indexer) getPointIDsByS2CellID(cellid s2.CellID) ([]spatialindexer.PointID, bool) { +func (indexer S2Indexer) getPointIDsByS2CellID(cellid s2.CellID) ([]spatialindexer.PointID, bool) { pointIDs, ok := indexer.cellID2PointIDs[cellid] return pointIDs, ok } diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go index 0cd26b65d75..2b7c5bddafb 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -20,12 +20,15 @@ func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) [ radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) region := s2.Region(s2.CapFromCenterAngle(center, radius)) cellUnion := regionCover.Covering(region) + return ([]s2.CellID)(cellUnion) } -func queryNearByPoints(indexer *s2Indexer, point spatialindexer.Location, radius float64) []spatialindexer.PointInfo { +func queryNearByPoints(indexer *S2Indexer, point spatialindexer.Location, radius float64) []spatialindexer.PointInfo { var result []spatialindexer.PointInfo + cellIDs := queryNearByS2Cells(point, radius) + for _, cellID := range cellIDs { pointIDs, hasCellID := indexer.getPointIDsByS2CellID(cellID) if !hasCellID { @@ -35,7 +38,7 @@ func queryNearByPoints(indexer *s2Indexer, point spatialindexer.Location, radius for _, pointID := range pointIDs { location, hasPointID := indexer.getPointLocationByPointID(pointID) if !hasPointID { - glog.Errorf("In queryNearByPoints, use incorrect pointID %v to query s2Indexer\n", pointID) + glog.Errorf("In queryNearByPoints, use incorrect pointID %v to query S2Indexer\n", pointID) continue } @@ -52,10 +55,12 @@ func queryNearByPoints(indexer *s2Indexer, point spatialindexer.Location, radius func generateDebugInfo4CellIDs(cellIDs []s2.CellID) { glog.Info("=================================\n") glog.Info("generateDebugInfo4CellIDs\n") + for _, cellID := range cellIDs { glog.Infof("CellID value = %d(uint64), string = %v, token = %s, level = %d\n", (uint64)(cellID), cellID, cellID.ToToken(), cellID.Level()) } + glog.Info("=================================\n") } @@ -71,5 +76,6 @@ func generateDebugURL(cellIDs []s2.CellID) string { for _, cellID := range cellIDs { url += cellID.ToToken() + "," } + return url } diff --git a/integration/service/spatialindexer/s2indexer/spatial_query_test.go b/integration/service/spatialindexer/s2indexer/spatial_query_test.go index 2b9599610a0..8926bc11845 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query_test.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query_test.go @@ -10,7 +10,7 @@ import ( ) func TestSpatialIndexQuery1(t *testing.T) { - fakeIndexer1 := s2Indexer{ + fakeIndexer1 := S2Indexer{ cellID2PointIDs: map[s2.CellID][]spatialindexer.PointID{ 9263834064756932608: {1, 2}, // 4/0010133 9263851656942977024: {1, 2}, // 4/00101332 From 705baecfb6565bb728813f224a67765e77a52600 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Thu, 26 Mar 2020 18:11:46 -0700 Subject: [PATCH 09/27] feat: implement feature of place connectivity issue: https://github.com/Telenav/osrm-backend/issues/238 --- .../chargestations-connectivity-gen/doc.go | 17 -------- .../chargestations-connectivity-gen/main.go | 10 ----- .../placeconnectivitymap/connectivity_map.go | 32 +++++++++++++++ .../service/placeconnectivitymap/doc.go | 31 ++++++++++++++ .../service/spatialindexer/interface.go | 16 +++++++- .../service/spatialindexer/ranker/doc.go | 4 ++ .../spatialindexer/ranker/rank_agent.go | 41 +++++++++++++++++++ .../ranker/rank_by_great_circle_distance.go | 30 ++++++++++++++ .../ranker/rank_by_osrm_shortest_path.go | 23 +++++++++++ .../spatialindexer/ranker/ranker_factory.go | 22 ++++++++++ .../spatialindexer/ranker/simple_ranker.go | 14 +++++++ .../spatialindexer/s2indexer/indexer.go | 35 ++++++++++++++++ 12 files changed, 246 insertions(+), 29 deletions(-) delete mode 100644 integration/cmd/chargestations-connectivity-gen/doc.go delete mode 100644 integration/cmd/chargestations-connectivity-gen/main.go create mode 100644 integration/service/placeconnectivitymap/connectivity_map.go create mode 100644 integration/service/placeconnectivitymap/doc.go create mode 100644 integration/service/spatialindexer/ranker/doc.go create mode 100644 integration/service/spatialindexer/ranker/rank_agent.go create mode 100644 integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go create mode 100644 integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go create mode 100644 integration/service/spatialindexer/ranker/ranker_factory.go create mode 100644 integration/service/spatialindexer/ranker/simple_ranker.go diff --git a/integration/cmd/chargestations-connectivity-gen/doc.go b/integration/cmd/chargestations-connectivity-gen/doc.go deleted file mode 100644 index 8e0f8b31e38..00000000000 --- a/integration/cmd/chargestations-connectivity-gen/doc.go +++ /dev/null @@ -1,17 +0,0 @@ -// package main contains the tool of chargestation-connectivity generator -// stage 1: -// inputs is json file -// => convert to slice of [id:string,location: lat,lon] -// => calculate cellids for each point(for all levels) -// => build revese index for cellid -> ids -// stage 2: -// => iterate each point -// => generate a circle(s2::cap), find all cellids intersect with that circle -// => retrieve all ids -// => generate result of id(from), ids(all ids in certain distance) -// stage 3: -// => load data from file -// => for each line, its formid and all other ids -// => calculate distance between fromid and all other ids -// => sort result based on distance and write back to file -package main diff --git a/integration/cmd/chargestations-connectivity-gen/main.go b/integration/cmd/chargestations-connectivity-gen/main.go deleted file mode 100644 index bd1acb5d084..00000000000 --- a/integration/cmd/chargestations-connectivity-gen/main.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -import ( - "fmt" -) - -func main() { - // @todo: add logic to generate connectivity for charge stations - fmt.Print("Hello World!") -} diff --git a/integration/service/placeconnectivitymap/connectivity_map.go b/integration/service/placeconnectivitymap/connectivity_map.go new file mode 100644 index 00000000000..49449ec013a --- /dev/null +++ b/integration/service/placeconnectivitymap/connectivity_map.go @@ -0,0 +1,32 @@ +package placeconnectivitymap + +import ( + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer/s2indexer" + "github.com/golang/glog" +) + +type PlaceConnectivityMap struct { + finder spatialindexer.Finder + ranker spatialindexer.Ranker + iterator spatialindexer.PointsIterator +} + +func NewPlaceConnectivityMap(filePath string) *PlaceConnectivityMap { + indexer := s2indexer.NewS2Indexer().Build(filePath) + if indexer == nil { + glog.Error("Failed to NewPlaceConnectivityMap due to empty indexer is generated. Check your input\n") + return nil + } + + // return &PlaceConnectivityMap{ + // finder: indexer, + // ranker: indexer, + // iterator: indexer, + // } + return nil +} + +func (pcm *PlaceConnectivityMap) GenerateConnectivityMap(filePath string) { + glog.Info("Successfully finished GenerateConnectivityMap\n") +} diff --git a/integration/service/placeconnectivitymap/doc.go b/integration/service/placeconnectivitymap/doc.go new file mode 100644 index 00000000000..5d0cd780597 --- /dev/null +++ b/integration/service/placeconnectivitymap/doc.go @@ -0,0 +1,31 @@ +/* +package placeconnectivitymap provides connectivity information for place graphs. + +For example, given following graph + - - - - 4 +| | | | | + - 2 - - - +| | | | | + - - - 3 - +| | | | | +1 - - - - + +Place connectivity map will pre-process each points and generate following result: +While query connectivity for place 1, will return + (place 2, 3), // the shortest path between place 1 and place 2 is 3 + (place 3, 4), // the shortest path between place 1 and place 2 is 4 + (place 4, 7), // the shortest path between place 1 and place 2 is 7 +The result is sorted by shortest path distance(or other user defined strategy) + +When query for connectivity, user could also pass in limitation option, such as distance limitation. +For example, when query connectivity for place 3 +With limitation = -1, it will return + (place 2, 3), // the shortest path between place 3 and place 2 is 3 + (place 4, 3), // the shortest path between place 3 and place 4 is 3 + (place 1, 4), // the shortest path between place 3 and place 1 is 4 +With limitation = 3, it will return + (place 2, 3), // the shortest path between place 3 and place 2 is 3 + (place 4, 3), // the shortest path between place 3 and place 4 is 3 + +*/ +package placeconnectivitymap diff --git a/integration/service/spatialindexer/interface.go b/integration/service/spatialindexer/interface.go index 7d0aaf5138c..0f5ea230551 100644 --- a/integration/service/spatialindexer/interface.go +++ b/integration/service/spatialindexer/interface.go @@ -1,5 +1,7 @@ package spatialindexer +import "math" + // Location for poi point // @todo: will be replaced by the one in map type Location struct { @@ -23,6 +25,9 @@ type RankedPointInfo struct { // Only the data used for pre-processing contains valid PointID type PointID int64 +// UnlimitedCount indicate all spatial search result will be returned +const UnlimitedCount = math.MaxInt32 + // Finder answers special query type Finder interface { @@ -34,8 +39,15 @@ type Finder interface { type Ranker interface { // RankPointIDsByGreatCircleDistance ranks a group of points based on great circle distance to given location - RankPointIDsByGreatCircleDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo + RankPointIDsByGreatCircleDistance(center Location, nearByIDs []*PointInfo) []*RankedPointInfo // RankPointIDsByShortestDistance ranks a group of points based on shortest path distance to given location - RankPointIDsByShortestDistance(center Location, nearByIDs []PointInfo) []RankedPointInfo + RankPointIDsByShortestDistance(center Location, nearByIDs []*PointInfo) []*RankedPointInfo +} + +// PointsIterator provides iterateability for PointInfo +type PointsIterator interface { + + // IteratePoints returns channel for PointInfo + IteratePoints() <-chan PointInfo } diff --git a/integration/service/spatialindexer/ranker/doc.go b/integration/service/spatialindexer/ranker/doc.go new file mode 100644 index 00000000000..d1ca5c5439c --- /dev/null +++ b/integration/service/spatialindexer/ranker/doc.go @@ -0,0 +1,4 @@ +/* +Package ranker implements interfaces defined in Ranker. +*/ +package ranker diff --git a/integration/service/spatialindexer/ranker/rank_agent.go b/integration/service/spatialindexer/ranker/rank_agent.go new file mode 100644 index 00000000000..a121a6d5ad7 --- /dev/null +++ b/integration/service/spatialindexer/ranker/rank_agent.go @@ -0,0 +1,41 @@ +package ranker + +import ( + "sort" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" +) + +type rankAgent struct { + rankedPoints []*spatialindexer.RankedPointInfo +} + +func newRankAgent(pointNum int) *rankAgent { + return &rankAgent{ + rankedPoints: make([]*spatialindexer.RankedPointInfo, 0, pointNum), + } +} + +type rankItems []*spatialindexer.RankedPointInfo + +func (r rankItems) Len() int { + return len(r) +} + +func (r rankItems) Less(i, j int) bool { + return r[i].Distance < r[j].Distance +} + +func (r rankItems) Swap(i, j int) { + r[i], r[j] = r[j], r[i] +} + +func (r *rankAgent) RankByDistance(input <-chan *spatialindexer.RankedPointInfo) []*spatialindexer.RankedPointInfo { + for p := range input { + r.rankedPoints = append(r.rankedPoints, p) + } + + sort.Sort(rankItems(r.rankedPoints)) + + return r.rankedPoints +} diff --git a/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go new file mode 100644 index 00000000000..ebaf7652e78 --- /dev/null +++ b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go @@ -0,0 +1,30 @@ +package ranker + +import ( + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/blevesearch/bleve/geo" + "github.com/golang/glog" +) + +func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + if len(nearByIDs) == 0 { + glog.Warning("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty\n") + return nil + } + + pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(nearByIDs)) + go func() { + for _, p := range nearByIDs { + pointWithDistanceC <- &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: p.ID, + Location: p.Location, + }, + Distance: geo.Haversin(center.Longitude, center.Latitude, p.Location.Longitude, p.Location.Latitude), + } + } + }() + + rankAgent := newRankAgent(len(nearByIDs)) + return rankAgent.RankByDistance(pointWithDistanceC) +} diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go new file mode 100644 index 00000000000..94f612ef3f7 --- /dev/null +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go @@ -0,0 +1,23 @@ +package ranker + +import ( + "github.com/Telenav/osrm-backend/integration/oasis/osrmconnector" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" +) + +// func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector) []*spatialindexer.RankedPointInfo { +// if len(nearByIDs) == 0 { +// glog.Warning("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty\n") +// return nil +// } + +// pointWithDistanceC := make([]*spatialindexer.RankedPointInfo, len(nearByIDs)) +// } + +func generateTableRequest(center spatialindexer.Location, + nearByIDs []*spatialindexer.PointInfo, + oc *osrmconnector.OSRMConnector, + outputC chan *spatialindexer.RankedPointInfo, + startIndex int) { + // todo +} diff --git a/integration/service/spatialindexer/ranker/ranker_factory.go b/integration/service/spatialindexer/ranker/ranker_factory.go new file mode 100644 index 00000000000..3b2fe63144b --- /dev/null +++ b/integration/service/spatialindexer/ranker/ranker_factory.go @@ -0,0 +1,22 @@ +package ranker + +import ( + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/Telenav/osrm-backend2/integration/oasis/osrmconnector" +) + +const ( + SimpleRanker = "SimpleRanker" + OSRMBasedRanker = "OSRMBasedRanker" +) + +func CreateRanker(rankerType string, oc *osrmconnector.OSRMConnector) spatialindexer.Ranker { + switch rankerType { + case SimpleRanker: + return nil + case OSRMBasedRanker: + return nil + default: + return nil + } +} diff --git a/integration/service/spatialindexer/ranker/simple_ranker.go b/integration/service/spatialindexer/ranker/simple_ranker.go new file mode 100644 index 00000000000..6b0bdc74ff6 --- /dev/null +++ b/integration/service/spatialindexer/ranker/simple_ranker.go @@ -0,0 +1,14 @@ +package ranker + +import "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + +type simpleRanker struct { +} + +func (ranker *simpleRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + return rankPointsByGreatCircleDistanceToCenter(center, nearByIDs) +} + +func (ranker *simpleRanker) RankPointIDsByShortestDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + return ranker.RankPointIDsByGreatCircleDistance(center, nearByIDs) +} diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go index 6806555078c..fe9e70635e1 100644 --- a/integration/service/spatialindexer/s2indexer/indexer.go +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -63,6 +63,37 @@ func (indexer *S2Indexer) Dump(folderPath string) { } } +// IteratePoints returns PointInfo in channel +// It implements interface of PointsIterator +func (indexer *S2Indexer) IteratePoints() <-chan spatialindexer.PointInfo { + pointsC := make(chan spatialindexer.PointInfo, len(indexer.pointID2Location)) + go func() { + for pointID, location := range indexer.pointID2Location { + pointsC <- spatialindexer.PointInfo{ + ID: pointID, + Location: location, + } + } + close(pointsC) + }() + + return pointsC +} + +func (indexer *S2Indexer) FindNearByPointIDs(center spatialindexer.Location, radius float64, limitCount int) []spatialindexer.PointInfo { + if !indexer.isInit() { + glog.Warning("S2Indexer is empty, try to Build() with correct input file first.\n") + return nil + } + + results := queryNearByPoints(indexer, center, radius) + if limitCount != spatialindexer.UnlimitedCount && len(results) > limitCount { + results = results[:limitCount] + } + + return results +} + func (indexer S2Indexer) getPointLocationByPointID(id spatialindexer.PointID) (spatialindexer.Location, bool) { location, ok := indexer.pointID2Location[id] return location, ok @@ -73,6 +104,10 @@ func (indexer S2Indexer) getPointIDsByS2CellID(cellid s2.CellID) ([]spatialindex return pointIDs, ok } +func (indexer S2Indexer) isInit() bool { + return len(indexer.cellID2PointIDs) != 0 && len(indexer.pointID2Location) != 0 +} + func elementID2PointID(id int64) spatialindexer.PointID { return (spatialindexer.PointID)(id) } From 7432b794885fab772a001dbbf2862d4da403fd0b Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Mon, 30 Mar 2020 10:22:26 -0700 Subject: [PATCH 10/27] fix: fix compilation issue. --- integration/go.mod | 1 + integration/go.sum | 2 ++ integration/service/spatialindexer/doc.go | 2 +- integration/service/spatialindexer/ranker/ranker_factory.go | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integration/go.mod b/integration/go.mod index 415bc0084aa..c005cfad0a4 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -3,6 +3,7 @@ module github.com/Telenav/osrm-backend/integration go 1.13 require ( + github.com/blevesearch/bleve v0.8.2 github.com/golang/geo v0.0.0-20200319012246-673a6f80352d github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.3.2 diff --git a/integration/go.sum b/integration/go.sum index 4eaede4a194..efe2c9a7d57 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/blevesearch/bleve v0.8.2 h1:dlG2QtAM91QzMatFLJAooZ8jpL/nG0xIA8+J0RHfcUY= +github.com/blevesearch/bleve v0.8.2/go.mod h1:Y2lmIkzV6mcNfAnAdOd+ZxHkHchhBfU/xroGIp61wfw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/golang/geo v0.0.0-20200319012246-673a6f80352d h1:C/hKUcHT483btRbeGkrRjJz+Zbcj8audldIi9tRJDCc= github.com/golang/geo v0.0.0-20200319012246-673a6f80352d/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= diff --git a/integration/service/spatialindexer/doc.go b/integration/service/spatialindexer/doc.go index 87b49a73af4..76ae07fa799 100644 --- a/integration/service/spatialindexer/doc.go +++ b/integration/service/spatialindexer/doc.go @@ -1,4 +1,4 @@ -// package spatialindexer answers query of nearest points(place, point of interest) for conditions +// Package spatialindexer answers query of nearest points(place, point of interest) for conditions // such as center location, radius, etc // // Sample Scenario 1: Build connectivity for charge stations during pro-processing diff --git a/integration/service/spatialindexer/ranker/ranker_factory.go b/integration/service/spatialindexer/ranker/ranker_factory.go index 3b2fe63144b..ba44eb304dc 100644 --- a/integration/service/spatialindexer/ranker/ranker_factory.go +++ b/integration/service/spatialindexer/ranker/ranker_factory.go @@ -1,8 +1,8 @@ package ranker import ( + "github.com/Telenav/osrm-backend/integration/oasis/osrmconnector" "github.com/Telenav/osrm-backend/integration/service/spatialindexer" - "github.com/Telenav/osrm-backend2/integration/oasis/osrmconnector" ) const ( From 5625883b9d973b325bdf723868693133b9d6db73 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Mon, 30 Mar 2020 13:59:00 -0700 Subject: [PATCH 11/27] fix: implement logic for ranker. --- .../service/spatialindexer/interface.go | 4 +- .../ranker/rank_by_great_circle_distance.go | 2 +- .../ranker/rank_by_osrm_shortest_path.go | 132 ++++++++++++++++-- .../spatialindexer/s2indexer/builder.go | 2 +- .../spatialindexer/s2indexer/builder_test.go | 28 ++-- .../spatialindexer/s2indexer/dumper_test.go | 12 +- .../spatialindexer/s2indexer/indexer.go | 8 +- .../spatialindexer/s2indexer/spatial_query.go | 2 +- .../s2indexer/spatial_query_test.go | 24 ++-- 9 files changed, 158 insertions(+), 56 deletions(-) diff --git a/integration/service/spatialindexer/interface.go b/integration/service/spatialindexer/interface.go index 0f5ea230551..a4149f59080 100644 --- a/integration/service/spatialindexer/interface.go +++ b/integration/service/spatialindexer/interface.go @@ -5,8 +5,8 @@ import "math" // Location for poi point // @todo: will be replaced by the one in map type Location struct { - Latitude float64 - Longitude float64 + Lat float64 + Lon float64 } // PointInfo records point related information such as ID and location diff --git a/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go index ebaf7652e78..764aad5d133 100644 --- a/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go +++ b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go @@ -20,7 +20,7 @@ func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, nea ID: p.ID, Location: p.Location, }, - Distance: geo.Haversin(center.Longitude, center.Latitude, p.Location.Longitude, p.Location.Latitude), + Distance: geo.Haversin(center.Lon, center.Lat, p.Location.Lon, p.Location.Lat), } } }() diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go index 94f612ef3f7..5210e477faf 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go @@ -1,23 +1,125 @@ package ranker import ( + "strconv" + "sync" + "github.com/Telenav/osrm-backend/integration/oasis/osrmconnector" + "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/coordinate" + "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/table" "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/glog" ) -// func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector) []*spatialindexer.RankedPointInfo { -// if len(nearByIDs) == 0 { -// glog.Warning("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty\n") -// return nil -// } - -// pointWithDistanceC := make([]*spatialindexer.RankedPointInfo, len(nearByIDs)) -// } - -func generateTableRequest(center spatialindexer.Location, - nearByIDs []*spatialindexer.PointInfo, - oc *osrmconnector.OSRMConnector, - outputC chan *spatialindexer.RankedPointInfo, - startIndex int) { - // todo +// pointsLimit4SingleTableRequest defines point count limitation for each single table request. +// During pre-processing, its possible to have situation to calculate distance between thousnads of points. +// The situation here is 1-to-N table request, use pointsLimit4SingleTableRequest to limit N +const pointsLimit4SingleTableRequest = 1000 + +func rankPointsByOSRMShortestPath(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector) []*spatialindexer.RankedPointInfo { + if len(nearByIDs) == 0 { + glog.Warning("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty\n") + return nil + } + + var wg sync.WaitGroup + pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(nearByIDs)) + startIndex := 0 + endIndex := 0 + for { + if startIndex >= len(nearByIDs) { + break + } + endIndex = startIndex + pointsLimit4SingleTableRequest + if endIndex >= len(nearByIDs) { + endIndex = len(nearByIDs) - 1 + } + + go func(wg *sync.WaitGroup) { + wg.Add(1) + defer wg.Done() + + rankedPoints, err := calcShortestPathDistance(center, nearByIDs, oc, startIndex, endIndex) + + if err != nil { + // @todo: add retry logic + } else { + for _, item := range rankedPoints { + pointWithDistanceC <- item + } + } + + }(&wg) + + startIndex = endIndex + 1 + } + + wg.Wait() + rankAgent := newRankAgent(len(nearByIDs)) + return rankAgent.RankByDistance(pointWithDistanceC) + +} + +func calcShortestPathDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector, startIndex, endIndex int) ([]*spatialindexer.RankedPointInfo, error) { + req := generateTableRequest(center, nearByIDs, startIndex, endIndex) + respC := oc.Request4Table(req) + resp := <-respC + + if resp.Err != nil { + glog.Error("Failed to generate table response for \n %s with \n err =%v \n", req.RequestURI, resp.Err) + return nil, resp.Err + } + + result := make([]*spatialindexer.RankedPointInfo, 0, endIndex-startIndex+1) + for i := 0; i < endIndex-startIndex+1; i++ { + result = append(result, &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: nearByIDs[startIndex+i].ID, + Location: nearByIDs[startIndex+i].Location, + }, + Distance: *resp.Resp.Distances[0][i], + }) + } + return result, nil +} + +// generateTableRequest generates table requests from center to [startIndex, endIndex] of nearByIDs +func generateTableRequest(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, startIndex, endIndex int) *table.Request { + if startIndex < 0 || startIndex > endIndex || endIndex >= len(nearByIDs) { + glog.Fatal("startIndex should be smaller equal to endIndex and both of them should in the range of len(nearByIDs), while (startIndex, endIndex, len(nearByIDs)) = (%d, %d, %d)", + startIndex, endIndex, len(nearByIDs)) + } + + req := table.NewRequest() + req.Coordinates = append(ConvertLocation2Coordinates(center), + ConvertPointInfos2Coordinates(nearByIDs, startIndex, endIndex)...) + + req.Sources = append(req.Sources, strconv.Itoa(0)) + pointsCount4Sources := 1 + for i := startIndex; i < endIndex; i++ { + str := strconv.Itoa(endIndex - startIndex + pointsCount4Sources) + req.Destinations = append(req.Destinations, str) + } + + return req +} + +func ConvertLocation2Coordinates(location spatialindexer.Location) coordinate.Coordinates { + result := make(coordinate.Coordinates, 0, 1) + result = append(result, coordinate.Coordinate{ + Lat: location.Lat, + Lon: location.Lon, + }) + return result +} + +func ConvertPointInfos2Coordinates(nearByIDs []*spatialindexer.PointInfo, startIndex, endIndex int) coordinate.Coordinates { + result := make(coordinate.Coordinates, 0, endIndex-startIndex+1) + for i := startIndex; i <= endIndex; i++ { + result = append(result, coordinate.Coordinate{ + Lat: nearByIDs[i].Location.Lat, + Lon: nearByIDs[i].Location.Lon, + }) + } + return result } diff --git a/integration/service/spatialindexer/s2indexer/builder.go b/integration/service/spatialindexer/s2indexer/builder.go index 49550b7ef47..00afb74c3fc 100644 --- a/integration/service/spatialindexer/s2indexer/builder.go +++ b/integration/service/spatialindexer/s2indexer/builder.go @@ -19,7 +19,7 @@ func build(points []spatialindexer.PointInfo, minLevel, maxLevel int) map[s2.Cel cellID2PointIDs := make(map[s2.CellID][]spatialindexer.PointID) for _, p := range points { - leafCellID := s2.CellFromLatLng(s2.LatLngFromDegrees(p.Location.Latitude, p.Location.Longitude)).ID() + leafCellID := s2.CellFromLatLng(s2.LatLngFromDegrees(p.Location.Lat, p.Location.Lon)).ID() var cellIDs []s2.CellID // For level = 30, its parent equal to current diff --git a/integration/service/spatialindexer/s2indexer/builder_test.go b/integration/service/spatialindexer/s2indexer/builder_test.go index 92b5c1a7502..7f86acc91cb 100644 --- a/integration/service/spatialindexer/s2indexer/builder_test.go +++ b/integration/service/spatialindexer/s2indexer/builder_test.go @@ -20,8 +20,8 @@ func TestBuild(t *testing.T) { spatialindexer.PointInfo{ ID: 1, Location: spatialindexer.Location{ - Latitude: 37.402701, - Longitude: -121.974096, + Lat: 37.402701, + Lon: -121.974096, }, }, }, @@ -59,15 +59,15 @@ func TestBuild(t *testing.T) { spatialindexer.PointInfo{ ID: 1, Location: spatialindexer.Location{ - Latitude: 37.402701, - Longitude: -121.974096, + Lat: 37.402701, + Lon: -121.974096, }, }, spatialindexer.PointInfo{ ID: 2, Location: spatialindexer.Location{ - Latitude: 37.403530, - Longitude: -121.969768, + Lat: 37.403530, + Lon: -121.969768, }, }, }, @@ -125,29 +125,29 @@ func TestBuild(t *testing.T) { spatialindexer.PointInfo{ ID: 1, Location: spatialindexer.Location{ - Latitude: 37.651275, - Longitude: -122.413744, + Lat: 37.651275, + Lon: -122.413744, }, }, spatialindexer.PointInfo{ ID: 2, Location: spatialindexer.Location{ - Latitude: 36.776215, - Longitude: -121.733663, + Lat: 36.776215, + Lon: -121.733663, }, }, spatialindexer.PointInfo{ ID: 3, Location: spatialindexer.Location{ - Latitude: 36.122438, - Longitude: -121.022936, + Lat: 36.122438, + Lon: -121.022936, }, }, spatialindexer.PointInfo{ ID: 4, Location: spatialindexer.Location{ - Latitude: 35.365543, - Longitude: -120.850000, + Lat: 35.365543, + Lon: -120.850000, }, }, }, diff --git a/integration/service/spatialindexer/s2indexer/dumper_test.go b/integration/service/spatialindexer/s2indexer/dumper_test.go index 5fd860dae76..6ac69d16df7 100644 --- a/integration/service/spatialindexer/s2indexer/dumper_test.go +++ b/integration/service/spatialindexer/s2indexer/dumper_test.go @@ -20,16 +20,16 @@ func TestDumper(t *testing.T) { }, pointID2Location: map[spatialindexer.PointID]spatialindexer.Location{ 1: spatialindexer.Location{ - Latitude: 11.11, - Longitude: 11.11, + Lat: 11.11, + Lon: 11.11, }, 2: spatialindexer.Location{ - Latitude: 22.22, - Longitude: 22.22, + Lat: 22.22, + Lon: 22.22, }, 3: spatialindexer.Location{ - Latitude: 33.33, - Longitude: 33.33, + Lat: 33.33, + Lon: 33.33, }, }, }, diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go index fe9e70635e1..2abee9646b6 100644 --- a/integration/service/spatialindexer/s2indexer/indexer.go +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -31,15 +31,15 @@ func (indexer *S2Indexer) Build(filePath string) *S2Indexer { pointInfo := spatialindexer.PointInfo{ ID: elementID2PointID(record.ID), Location: spatialindexer.Location{ - Latitude: record.Lat, - Longitude: record.Lon, + Lat: record.Lat, + Lon: record.Lon, }, } pointInfos = append(pointInfos, pointInfo) indexer.pointID2Location[elementID2PointID(record.ID)] = spatialindexer.Location{ - Latitude: record.Lat, - Longitude: record.Lon, + Lat: record.Lat, + Lon: record.Lon, } } diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go index 2b7c5bddafb..d64b997d284 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -16,7 +16,7 @@ func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) [ MinLevel: minS2Level, MaxLevel: maxS2Level, MaxCells: maxCellCount} - center := s2.PointFromLatLng(s2.LatLngFromDegrees(point.Latitude, point.Longitude)) + center := s2.PointFromLatLng(s2.LatLngFromDegrees(point.Lat, point.Lon)) radius := (s1.Angle)(radiusInMeters / s2EarthRadiusInMeters) region := s2.Region(s2.CapFromCenterAngle(center, radius)) cellUnion := regionCover.Covering(region) diff --git a/integration/service/spatialindexer/s2indexer/spatial_query_test.go b/integration/service/spatialindexer/s2indexer/spatial_query_test.go index 8926bc11845..c444a8f0d8a 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query_test.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query_test.go @@ -58,35 +58,35 @@ func TestSpatialIndexQuery1(t *testing.T) { }, pointID2Location: map[spatialindexer.PointID]spatialindexer.Location{ 1: spatialindexer.Location{ - Latitude: 37.402701, - Longitude: -121.974096, + Lat: 37.402701, + Lon: -121.974096, }, 2: spatialindexer.Location{ - Latitude: 37.403530, - Longitude: -121.969768, + Lat: 37.403530, + Lon: -121.969768, }, }, } // center in 4655 great america pkwy center := spatialindexer.Location{ - Latitude: 37.402799, - Longitude: -121.969861, + Lat: 37.402799, + Lon: -121.969861, } expect := []spatialindexer.PointInfo{ spatialindexer.PointInfo{ ID: 1, Location: spatialindexer.Location{ - Latitude: 37.402701, - Longitude: -121.974096, + Lat: 37.402701, + Lon: -121.974096, }, }, spatialindexer.PointInfo{ ID: 2, Location: spatialindexer.Location{ - Latitude: 37.40353, - Longitude: -121.969768, + Lat: 37.40353, + Lon: -121.969768, }, }, } @@ -102,8 +102,8 @@ func TestSpatialIndexQuery1(t *testing.T) { func TestQueryNearByS2Cells1(t *testing.T) { // center in 4655 great america pkwy center := spatialindexer.Location{ - Latitude: 37.402799, - Longitude: -121.969861, + Lat: 37.402799, + Lon: -121.969861, } actualCellIDs := queryNearByS2Cells(center, 1600) glog.Infof("\nTest URL is %s\n", generateDebugURL(actualCellIDs)) From 3d0d37491edc56537fafa80750c87b5582aae278 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Mon, 30 Mar 2020 15:23:56 -0700 Subject: [PATCH 12/27] fix: add unit tests for simple ranker --- .../spatialindexer/ranker/osrm_ranker.go | 24 ++++ .../spatialindexer/ranker/rank_agent_test.go | 136 ++++++++++++++++++ ... => rank_by_great_circle_distance_impl.go} | 5 +- ....go => rank_by_osrm_shortest_path_impl.go} | 8 +- .../spatialindexer/ranker/ranker_factory.go | 6 +- .../spatialindexer/ranker/simple_ranker.go | 4 + .../ranker/simple_ranker_test.go | 89 ++++++++++++ 7 files changed, 265 insertions(+), 7 deletions(-) create mode 100644 integration/service/spatialindexer/ranker/osrm_ranker.go create mode 100644 integration/service/spatialindexer/ranker/rank_agent_test.go rename integration/service/spatialindexer/ranker/{rank_by_great_circle_distance.go => rank_by_great_circle_distance_impl.go} (88%) rename integration/service/spatialindexer/ranker/{rank_by_osrm_shortest_path.go => rank_by_osrm_shortest_path_impl.go} (91%) create mode 100644 integration/service/spatialindexer/ranker/simple_ranker_test.go diff --git a/integration/service/spatialindexer/ranker/osrm_ranker.go b/integration/service/spatialindexer/ranker/osrm_ranker.go new file mode 100644 index 00000000000..f60b8ae6622 --- /dev/null +++ b/integration/service/spatialindexer/ranker/osrm_ranker.go @@ -0,0 +1,24 @@ +package ranker + +import ( + "github.com/Telenav/osrm-backend/integration/oasis/osrmconnector" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" +) + +type osrmRanker struct { + oc *osrmconnector.OSRMConnector +} + +func newOsrmRanker(oc *osrmconnector.OSRMConnector) *osrmRanker { + return &osrmRanker{ + oc: oc, + } +} + +func (ranker *osrmRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + return rankPointsByGreatCircleDistanceToCenter(center, nearByIDs) +} + +func (ranker *osrmRanker) RankPointIDsByShortestDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + return rankPointsByOSRMShortestPath(center, nearByIDs, ranker.oc) +} diff --git a/integration/service/spatialindexer/ranker/rank_agent_test.go b/integration/service/spatialindexer/ranker/rank_agent_test.go new file mode 100644 index 00000000000..8e5a951c351 --- /dev/null +++ b/integration/service/spatialindexer/ranker/rank_agent_test.go @@ -0,0 +1,136 @@ +package ranker + +import ( + "fmt" + "reflect" + "sync" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" +) + +func TestRankAgent(t *testing.T) { + cases := []struct { + input []*spatialindexer.RankedPointInfo + expect []*spatialindexer.RankedPointInfo + }{ + { + input: []*spatialindexer.RankedPointInfo{ + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 3.3, + Lon: 3.3, + }, + }, + Distance: 3.3, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 1.1, + Lon: 1.1, + }, + }, + Distance: 1.1, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 22, + Location: spatialindexer.Location{ + Lat: 22.22, + Lon: 22.22, + }, + }, + Distance: 22.22, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Lat: 4.4, + Lon: 4.4, + }, + }, + Distance: 4.4, + }, + }, + expect: []*spatialindexer.RankedPointInfo{ + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 1.1, + Lon: 1.1, + }, + }, + Distance: 1.1, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 3.3, + Lon: 3.3, + }, + }, + Distance: 3.3, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Lat: 4.4, + Lon: 4.4, + }, + }, + Distance: 4.4, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 22, + Location: spatialindexer.Location{ + Lat: 22.22, + Lon: 22.22, + }, + }, + Distance: 22.22, + }, + }, + }, + } + + for _, c := range cases { + var wg sync.WaitGroup + pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(c.input)) + go func(wg *sync.WaitGroup) { + wg.Add(1) + defer wg.Done() + + for _, item := range c.input { + pointWithDistanceC <- item + } + close(pointWithDistanceC) + }(&wg) + + wg.Wait() + rankAgent := newRankAgent(len(c.input)) + actual := rankAgent.RankByDistance(pointWithDistanceC) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("During test rankAgent, while handling \n input\n %s,\n expect\n %s \n but actual is\n %s\n", + printRankedPointInfoArray(c.input), + printRankedPointInfoArray(c.expect), + printRankedPointInfoArray(actual)) + } + } +} + +func printRankedPointInfoArray(arr []*spatialindexer.RankedPointInfo) string { + var str string + for _, item := range arr { + str += fmt.Sprintf("%#v ", item) + } + return str +} diff --git a/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go similarity index 88% rename from integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go rename to integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go index 764aad5d133..86d40a5019f 100644 --- a/integration/service/spatialindexer/ranker/rank_by_great_circle_distance.go +++ b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go @@ -14,13 +14,16 @@ func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, nea pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(nearByIDs)) go func() { + defer close(pointWithDistanceC) + for _, p := range nearByIDs { pointWithDistanceC <- &spatialindexer.RankedPointInfo{ PointInfo: spatialindexer.PointInfo{ ID: p.ID, Location: p.Location, }, - Distance: geo.Haversin(center.Lon, center.Lat, p.Location.Lon, p.Location.Lat), + // geo.Haversin's unit is kilometer, convert to meter + Distance: geo.Haversin(center.Lon, center.Lat, p.Location.Lon, p.Location.Lat) * 1000, } } }() diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go similarity index 91% rename from integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go rename to integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go index 5210e477faf..c9f94c65779 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go @@ -42,7 +42,7 @@ func rankPointsByOSRMShortestPath(center spatialindexer.Location, nearByIDs []*s rankedPoints, err := calcShortestPathDistance(center, nearByIDs, oc, startIndex, endIndex) if err != nil { - // @todo: add retry logic + // @todo: add retry logic when failed } else { for _, item := range rankedPoints { pointWithDistanceC <- item @@ -55,6 +55,8 @@ func rankPointsByOSRMShortestPath(center spatialindexer.Location, nearByIDs []*s } wg.Wait() + close(pointWithDistanceC) + rankAgent := newRankAgent(len(nearByIDs)) return rankAgent.RankByDistance(pointWithDistanceC) @@ -66,7 +68,7 @@ func calcShortestPathDistance(center spatialindexer.Location, nearByIDs []*spati resp := <-respC if resp.Err != nil { - glog.Error("Failed to generate table response for \n %s with \n err =%v \n", req.RequestURI, resp.Err) + glog.Errorf("Failed to generate table response for \n %s with \n err =%v \n", req.RequestURI(), resp.Err) return nil, resp.Err } @@ -86,7 +88,7 @@ func calcShortestPathDistance(center spatialindexer.Location, nearByIDs []*spati // generateTableRequest generates table requests from center to [startIndex, endIndex] of nearByIDs func generateTableRequest(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, startIndex, endIndex int) *table.Request { if startIndex < 0 || startIndex > endIndex || endIndex >= len(nearByIDs) { - glog.Fatal("startIndex should be smaller equal to endIndex and both of them should in the range of len(nearByIDs), while (startIndex, endIndex, len(nearByIDs)) = (%d, %d, %d)", + glog.Fatalf("startIndex should be smaller equal to endIndex and both of them should in the range of len(nearByIDs), while (startIndex, endIndex, len(nearByIDs)) = (%d, %d, %d)", startIndex, endIndex, len(nearByIDs)) } diff --git a/integration/service/spatialindexer/ranker/ranker_factory.go b/integration/service/spatialindexer/ranker/ranker_factory.go index ba44eb304dc..a6bd0776975 100644 --- a/integration/service/spatialindexer/ranker/ranker_factory.go +++ b/integration/service/spatialindexer/ranker/ranker_factory.go @@ -13,10 +13,10 @@ const ( func CreateRanker(rankerType string, oc *osrmconnector.OSRMConnector) spatialindexer.Ranker { switch rankerType { case SimpleRanker: - return nil + return newSimpleRanker() case OSRMBasedRanker: - return nil + return newOsrmRanker(oc) default: - return nil + return newSimpleRanker() } } diff --git a/integration/service/spatialindexer/ranker/simple_ranker.go b/integration/service/spatialindexer/ranker/simple_ranker.go index 6b0bdc74ff6..c765ce9f29e 100644 --- a/integration/service/spatialindexer/ranker/simple_ranker.go +++ b/integration/service/spatialindexer/ranker/simple_ranker.go @@ -5,6 +5,10 @@ import "github.com/Telenav/osrm-backend/integration/service/spatialindexer" type simpleRanker struct { } +func newSimpleRanker() *simpleRanker { + return &simpleRanker{} +} + func (ranker *simpleRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { return rankPointsByGreatCircleDistanceToCenter(center, nearByIDs) } diff --git a/integration/service/spatialindexer/ranker/simple_ranker_test.go b/integration/service/spatialindexer/ranker/simple_ranker_test.go new file mode 100644 index 00000000000..8fcf51ea616 --- /dev/null +++ b/integration/service/spatialindexer/ranker/simple_ranker_test.go @@ -0,0 +1,89 @@ +package ranker + +import ( + "reflect" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" +) + +func TestRankerInterfaceViaSimpleRanker(t *testing.T) { + cases := []struct { + center spatialindexer.Location + nearByIDs []*spatialindexer.PointInfo + expect []*spatialindexer.RankedPointInfo + }{ + { + center: spatialindexer.Location{ + Lat: 37.398973, + Lon: -121.976633, + }, + nearByIDs: []*spatialindexer.PointInfo{ + &spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 37.388840, + Lon: -121.981736, + }, + }, + &spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Lat: 37.375515, + Lon: -121.942812, + }, + }, + &spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 37.336954, + Lon: -121.861624, + }, + }, + }, + expect: []*spatialindexer.RankedPointInfo{ + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 37.388840, + Lon: -121.981736, + }, + }, + Distance: 1213.445757354474, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Lat: 37.375515, + Lon: -121.942812, + }, + }, + Distance: 3965.986474110687, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 37.336954, + Lon: -121.861624, + }, + }, + Distance: 12281.070927352637, + }, + }, + }, + } + + ranker := CreateRanker(SimpleRanker, nil) + for _, c := range cases { + actual := ranker.RankPointIDsByGreatCircleDistance(c.center, c.nearByIDs) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("During test SimpleRanker's RankPointIDsByGreatCircleDistance, \n expect \n%s \nwhile actual is\n %s\n", + printRankedPointInfoArray(c.expect), + printRankedPointInfoArray(actual)) + } + } + +} From 9f3c40d06b9d08ceeac3697676ee7ec5cdeb4b59 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Mon, 30 Mar 2020 15:53:28 -0700 Subject: [PATCH 13/27] fix: add unit tests for functions --- .../ranker/rank_by_osrm_shortest_path_impl.go | 16 +- .../rank_by_osrm_shortest_path_impl_test.go | 196 ++++++++++++++++++ .../ranker/simple_ranker_test.go | 7 + 3 files changed, 213 insertions(+), 6 deletions(-) create mode 100644 integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go index c9f94c65779..6b2d8249925 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go @@ -5,7 +5,9 @@ import ( "sync" "github.com/Telenav/osrm-backend/integration/oasis/osrmconnector" + "github.com/Telenav/osrm-backend/integration/pkg/api" "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/coordinate" + "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/route/options" "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/table" "github.com/Telenav/osrm-backend/integration/service/spatialindexer" "github.com/golang/glog" @@ -93,20 +95,22 @@ func generateTableRequest(center spatialindexer.Location, nearByIDs []*spatialin } req := table.NewRequest() - req.Coordinates = append(ConvertLocation2Coordinates(center), - ConvertPointInfos2Coordinates(nearByIDs, startIndex, endIndex)...) + req.Coordinates = append(convertLocation2Coordinates(center), + convertPointInfos2Coordinates(nearByIDs, startIndex, endIndex)...) req.Sources = append(req.Sources, strconv.Itoa(0)) pointsCount4Sources := 1 - for i := startIndex; i < endIndex; i++ { - str := strconv.Itoa(endIndex - startIndex + pointsCount4Sources) + for i := startIndex; i <= endIndex; i++ { + str := strconv.Itoa(i + pointsCount4Sources) req.Destinations = append(req.Destinations, str) } + req.Annotations = options.AnnotationsValueDistance + api.Comma + options.AnnotationsValueDuration + return req } -func ConvertLocation2Coordinates(location spatialindexer.Location) coordinate.Coordinates { +func convertLocation2Coordinates(location spatialindexer.Location) coordinate.Coordinates { result := make(coordinate.Coordinates, 0, 1) result = append(result, coordinate.Coordinate{ Lat: location.Lat, @@ -115,7 +119,7 @@ func ConvertLocation2Coordinates(location spatialindexer.Location) coordinate.Co return result } -func ConvertPointInfos2Coordinates(nearByIDs []*spatialindexer.PointInfo, startIndex, endIndex int) coordinate.Coordinates { +func convertPointInfos2Coordinates(nearByIDs []*spatialindexer.PointInfo, startIndex, endIndex int) coordinate.Coordinates { result := make(coordinate.Coordinates, 0, endIndex-startIndex+1) for i := startIndex; i <= endIndex; i++ { result = append(result, coordinate.Coordinate{ diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go new file mode 100644 index 00000000000..f2277837e1d --- /dev/null +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go @@ -0,0 +1,196 @@ +package ranker + +import ( + "reflect" + "testing" + + "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/coordinate" + "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/genericoptions" + "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/table" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" +) + +func TestGenerateTableRequest(t *testing.T) { + cases := []struct { + center spatialindexer.Location + nearByIDs []*spatialindexer.PointInfo + startIndex int + endIndex int + expect *table.Request + }{ + // case 1: test 0 -> {1, 2, 3, 4, 5} + { + center: spatialindexer.Location{ + Lat: 0, + Lon: 0, + }, + nearByIDs: []*spatialindexer.PointInfo{ + &spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 1.1, + Lon: 1.1, + }, + }, + &spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Lat: 2.2, + Lon: 2.2, + }, + }, + &spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 3.3, + Lon: 3.3, + }, + }, + &spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Lat: 4.4, + Lon: 4.4, + }, + }, + &spatialindexer.PointInfo{ + ID: 5, + Location: spatialindexer.Location{ + Lat: 5.5, + Lon: 5.5, + }, + }, + }, + startIndex: 0, + endIndex: 4, + expect: &table.Request{ + Service: "table", + Version: "v1", + Profile: "driving", + Coordinates: coordinate.Coordinates{ + coordinate.Coordinate{ + Lat: 0, + Lon: 0, + }, + coordinate.Coordinate{ + Lat: 1.1, + Lon: 1.1, + }, + coordinate.Coordinate{ + Lat: 2.2, + Lon: 2.2, + }, + coordinate.Coordinate{ + Lat: 3.3, + Lon: 3.3, + }, + coordinate.Coordinate{ + Lat: 4.4, + Lon: 4.4, + }, + coordinate.Coordinate{ + Lat: 5.5, + Lon: 5.5, + }, + }, + Sources: genericoptions.Elements{ + "0", + }, + Destinations: genericoptions.Elements{ + "1", + "2", + "3", + "4", + "5", + }, + Annotations: "distance,duration", + }, + }, + // case 2: test 0 -> {2, 3, 4} + { + center: spatialindexer.Location{ + Lat: 0, + Lon: 0, + }, + nearByIDs: []*spatialindexer.PointInfo{ + &spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 1.1, + Lon: 1.1, + }, + }, + &spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Lat: 2.2, + Lon: 2.2, + }, + }, + &spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 3.3, + Lon: 3.3, + }, + }, + &spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Lat: 4.4, + Lon: 4.4, + }, + }, + &spatialindexer.PointInfo{ + ID: 5, + Location: spatialindexer.Location{ + Lat: 5.5, + Lon: 5.5, + }, + }, + }, + startIndex: 1, + endIndex: 3, + expect: &table.Request{ + Service: "table", + Version: "v1", + Profile: "driving", + Coordinates: coordinate.Coordinates{ + coordinate.Coordinate{ + Lat: 0, + Lon: 0, + }, + coordinate.Coordinate{ + Lat: 2.2, + Lon: 2.2, + }, + coordinate.Coordinate{ + Lat: 3.3, + Lon: 3.3, + }, + coordinate.Coordinate{ + Lat: 4.4, + Lon: 4.4, + }, + }, + Sources: genericoptions.Elements{ + "0", + }, + Destinations: genericoptions.Elements{ + "2", + "3", + "4", + }, + Annotations: "distance,duration", + }, + }, + } + + for _, c := range cases { + actual := generateTableRequest(c.center, c.nearByIDs, c.startIndex, c.endIndex) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("During TestGenerateTableRequest, expect table request is \n%+v\n but actual is \n%+v\n", c.expect, actual) + } + } + +} diff --git a/integration/service/spatialindexer/ranker/simple_ranker_test.go b/integration/service/spatialindexer/ranker/simple_ranker_test.go index 8fcf51ea616..1dce8bcdd2c 100644 --- a/integration/service/spatialindexer/ranker/simple_ranker_test.go +++ b/integration/service/spatialindexer/ranker/simple_ranker_test.go @@ -84,6 +84,13 @@ func TestRankerInterfaceViaSimpleRanker(t *testing.T) { printRankedPointInfoArray(c.expect), printRankedPointInfoArray(actual)) } + + actual = ranker.RankPointIDsByGreatCircleDistance(c.center, c.nearByIDs) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("During test SimpleRanker's RankPointIDsByGreatCircleDistance, \n expect \n%s \nwhile actual is\n %s\n", + printRankedPointInfoArray(c.expect), + printRankedPointInfoArray(actual)) + } } } From b6ba5c6c61200b91c9f21661d291a56f4667fac1 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Mon, 30 Mar 2020 18:17:39 -0700 Subject: [PATCH 14/27] fix: add unit test for shortest_path_impl --- .../placeconnectivitymap/connectivity_map.go | 5 - .../service/placeconnectivitymap/doc.go | 1 + .../service/spatialindexer/interface.go | 4 +- .../spatialindexer/ranker/osrm_ranker.go | 8 +- .../spatialindexer/ranker/osrm_ranker_test.go | 262 ++++++++++++++++++ .../rank_by_great_circle_distance_impl.go | 10 +- .../ranker/rank_by_osrm_shortest_path_impl.go | 59 ++-- .../rank_by_osrm_shortest_path_impl_test.go | 197 ++++++++++++- .../spatialindexer/ranker/simple_ranker.go | 8 +- .../ranker/simple_ranker_test.go | 12 +- 10 files changed, 505 insertions(+), 61 deletions(-) create mode 100644 integration/service/spatialindexer/ranker/osrm_ranker_test.go diff --git a/integration/service/placeconnectivitymap/connectivity_map.go b/integration/service/placeconnectivitymap/connectivity_map.go index 49449ec013a..4bd4b51337f 100644 --- a/integration/service/placeconnectivitymap/connectivity_map.go +++ b/integration/service/placeconnectivitymap/connectivity_map.go @@ -19,11 +19,6 @@ func NewPlaceConnectivityMap(filePath string) *PlaceConnectivityMap { return nil } - // return &PlaceConnectivityMap{ - // finder: indexer, - // ranker: indexer, - // iterator: indexer, - // } return nil } diff --git a/integration/service/placeconnectivitymap/doc.go b/integration/service/placeconnectivitymap/doc.go index 5d0cd780597..a9fa85a83c9 100644 --- a/integration/service/placeconnectivitymap/doc.go +++ b/integration/service/placeconnectivitymap/doc.go @@ -2,6 +2,7 @@ package placeconnectivitymap provides connectivity information for place graphs. For example, given following graph + - - - - 4 | | | | | - 2 - - - diff --git a/integration/service/spatialindexer/interface.go b/integration/service/spatialindexer/interface.go index a4149f59080..ef347245135 100644 --- a/integration/service/spatialindexer/interface.go +++ b/integration/service/spatialindexer/interface.go @@ -39,10 +39,10 @@ type Finder interface { type Ranker interface { // RankPointIDsByGreatCircleDistance ranks a group of points based on great circle distance to given location - RankPointIDsByGreatCircleDistance(center Location, nearByIDs []*PointInfo) []*RankedPointInfo + RankPointIDsByGreatCircleDistance(center Location, targets []*PointInfo) []*RankedPointInfo // RankPointIDsByShortestDistance ranks a group of points based on shortest path distance to given location - RankPointIDsByShortestDistance(center Location, nearByIDs []*PointInfo) []*RankedPointInfo + RankPointIDsByShortestDistance(center Location, targets []*PointInfo) []*RankedPointInfo } // PointsIterator provides iterateability for PointInfo diff --git a/integration/service/spatialindexer/ranker/osrm_ranker.go b/integration/service/spatialindexer/ranker/osrm_ranker.go index f60b8ae6622..72184fe64d6 100644 --- a/integration/service/spatialindexer/ranker/osrm_ranker.go +++ b/integration/service/spatialindexer/ranker/osrm_ranker.go @@ -15,10 +15,10 @@ func newOsrmRanker(oc *osrmconnector.OSRMConnector) *osrmRanker { } } -func (ranker *osrmRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { - return rankPointsByGreatCircleDistanceToCenter(center, nearByIDs) +func (ranker *osrmRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + return rankPointsByGreatCircleDistanceToCenter(center, targets) } -func (ranker *osrmRanker) RankPointIDsByShortestDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { - return rankPointsByOSRMShortestPath(center, nearByIDs, ranker.oc) +func (ranker *osrmRanker) RankPointIDsByShortestDistance(center spatialindexer.Location, targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + return rankPointsByOSRMShortestPath(center, targets, ranker.oc, pointsThresholdPerRequest) } diff --git a/integration/service/spatialindexer/ranker/osrm_ranker_test.go b/integration/service/spatialindexer/ranker/osrm_ranker_test.go new file mode 100644 index 00000000000..802e515ff83 --- /dev/null +++ b/integration/service/spatialindexer/ranker/osrm_ranker_test.go @@ -0,0 +1,262 @@ +package ranker + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/Telenav/osrm-backend/integration/oasis/osrmconnector" + "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/table" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" +) + +var mockFloatArray1To6 []float64 = []float64{1.1, 2.2, 3.3, 4.4, 5.5, 6.6} + +var mock1To6TableResponse table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + &mockFloatArray1To6[3], + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + &mockFloatArray1To6[3], + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, +} + +var mock1To3TableResponsePart1 table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + }, + }, +} + +var mock1To3TableResponsePart2 table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[3], + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[3], + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, +} + +var mock1To4TableResponsePart1 table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + &mockFloatArray1To6[3], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + &mockFloatArray1To6[3], + }, + }, +} + +var mock1To4TableResponsePart2 table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, +} + +func TestRankerInterfaceViaOSRMRanker(t *testing.T) { + cases := []struct { + center spatialindexer.Location + targets []*spatialindexer.PointInfo + expect []*spatialindexer.RankedPointInfo + }{ + { + center: spatialindexer.Location{ + Lat: 0, + Lon: 0, + }, + targets: []*spatialindexer.PointInfo{ + &spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 1.1, + Lon: 1.1, + }, + }, + &spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Lat: 2.2, + Lon: 2.2, + }, + }, + &spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 3.3, + Lon: 3.3, + }, + }, + &spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Lat: 4.4, + Lon: 4.4, + }, + }, + &spatialindexer.PointInfo{ + ID: 5, + Location: spatialindexer.Location{ + Lat: 5.5, + Lon: 5.5, + }, + }, + &spatialindexer.PointInfo{ + ID: 6, + Location: spatialindexer.Location{ + Lat: 6.6, + Lon: 6.6, + }, + }, + }, + expect: []*spatialindexer.RankedPointInfo{ + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 1.1, + Lon: 1.1, + }, + }, + Distance: 1.1, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Lat: 2.2, + Lon: 2.2, + }, + }, + Distance: 2.2, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 3.3, + Lon: 3.3, + }, + }, + Distance: 3.3, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Lat: 4.4, + Lon: 4.4, + }, + }, + Distance: 4.4, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 5, + Location: spatialindexer.Location{ + Lat: 5.5, + Lon: 5.5, + }, + }, + Distance: 5.5, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 6, + Location: spatialindexer.Location{ + Lat: 6.6, + Lon: 6.6, + }, + }, + Distance: 6.6, + }, + }, + }, + } + + // fake OSRM server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + + if r.Method != "GET" { + t.Errorf("Expected 'GET' request, got '%s'", r.Method) + } + + if strings.HasPrefix(r.URL.EscapedPath(), "/table/v1/driving/") { + req, _ := table.ParseRequestURL(r.URL) + s := len(req.Sources) + d := len(req.Destinations) + if s == 1 && d == 6 { + var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To6TableResponse) + w.Write(tableResponseBytesOrig2Location1) + } + return + } + + })) + defer ts.Close() + + oc := osrmconnector.NewOSRMConnector(ts.URL) + ranker := CreateRanker(OSRMBasedRanker, oc) + + for _, c := range cases { + actual := ranker.RankPointIDsByShortestDistance(c.center, c.targets) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("During TestRankerInterfaceViaOSRMRanker, expect \n%s \nwhile actual result is \n%s\n", + printRankedPointInfoArray(c.expect), + printRankedPointInfoArray(actual)) + } + } +} diff --git a/integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go index 86d40a5019f..9e6787a8503 100644 --- a/integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go +++ b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go @@ -6,17 +6,17 @@ import ( "github.com/golang/glog" ) -func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { - if len(nearByIDs) == 0 { +func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + if len(targets) == 0 { glog.Warning("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty\n") return nil } - pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(nearByIDs)) + pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(targets)) go func() { defer close(pointWithDistanceC) - for _, p := range nearByIDs { + for _, p := range targets { pointWithDistanceC <- &spatialindexer.RankedPointInfo{ PointInfo: spatialindexer.PointInfo{ ID: p.ID, @@ -28,6 +28,6 @@ func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, nea } }() - rankAgent := newRankAgent(len(nearByIDs)) + rankAgent := newRankAgent(len(targets)) return rankAgent.RankByDistance(pointWithDistanceC) } diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go index 6b2d8249925..1fd6ae64fc5 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go @@ -13,35 +13,34 @@ import ( "github.com/golang/glog" ) -// pointsLimit4SingleTableRequest defines point count limitation for each single table request. +// pointsThresholdPerRequest defines point count seperator for each single table request. // During pre-processing, its possible to have situation to calculate distance between thousnads of points. // The situation here is 1-to-N table request, use pointsLimit4SingleTableRequest to limit N -const pointsLimit4SingleTableRequest = 1000 +const pointsThresholdPerRequest = 1000 -func rankPointsByOSRMShortestPath(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector) []*spatialindexer.RankedPointInfo { - if len(nearByIDs) == 0 { +func rankPointsByOSRMShortestPath(center spatialindexer.Location, targets []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector, pointsThreshold int) []*spatialindexer.RankedPointInfo { + if len(targets) == 0 { glog.Warning("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty\n") return nil } var wg sync.WaitGroup - pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(nearByIDs)) + pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(targets)) startIndex := 0 endIndex := 0 for { - if startIndex >= len(nearByIDs) { + if startIndex >= len(targets) { break } - endIndex = startIndex + pointsLimit4SingleTableRequest - if endIndex >= len(nearByIDs) { - endIndex = len(nearByIDs) - 1 - } - go func(wg *sync.WaitGroup) { - wg.Add(1) - defer wg.Done() + endIndex = startIndex + pointsThreshold - 1 + if endIndex >= len(targets) { + endIndex = len(targets) - 1 + } - rankedPoints, err := calcShortestPathDistance(center, nearByIDs, oc, startIndex, endIndex) + wg.Add(1) + go func(wg *sync.WaitGroup, startIndex, endIndex int) { + rankedPoints, err := calcShortestPathDistance(center, targets, oc, startIndex, endIndex) if err != nil { // @todo: add retry logic when failed @@ -51,7 +50,8 @@ func rankPointsByOSRMShortestPath(center spatialindexer.Location, nearByIDs []*s } } - }(&wg) + wg.Done() + }(&wg, startIndex, endIndex) startIndex = endIndex + 1 } @@ -59,13 +59,13 @@ func rankPointsByOSRMShortestPath(center spatialindexer.Location, nearByIDs []*s wg.Wait() close(pointWithDistanceC) - rankAgent := newRankAgent(len(nearByIDs)) + rankAgent := newRankAgent(len(targets)) return rankAgent.RankByDistance(pointWithDistanceC) } -func calcShortestPathDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector, startIndex, endIndex int) ([]*spatialindexer.RankedPointInfo, error) { - req := generateTableRequest(center, nearByIDs, startIndex, endIndex) +func calcShortestPathDistance(center spatialindexer.Location, targets []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector, startIndex, endIndex int) ([]*spatialindexer.RankedPointInfo, error) { + req := generateTableRequest(center, targets, startIndex, endIndex) respC := oc.Request4Table(req) resp := <-respC @@ -73,13 +73,14 @@ func calcShortestPathDistance(center spatialindexer.Location, nearByIDs []*spati glog.Errorf("Failed to generate table response for \n %s with \n err =%v \n", req.RequestURI(), resp.Err) return nil, resp.Err } + glog.Infof("Inside ranker, get table response for request %s\n", req.RequestURI()) result := make([]*spatialindexer.RankedPointInfo, 0, endIndex-startIndex+1) for i := 0; i < endIndex-startIndex+1; i++ { result = append(result, &spatialindexer.RankedPointInfo{ PointInfo: spatialindexer.PointInfo{ - ID: nearByIDs[startIndex+i].ID, - Location: nearByIDs[startIndex+i].Location, + ID: targets[startIndex+i].ID, + Location: targets[startIndex+i].Location, }, Distance: *resp.Resp.Distances[0][i], }) @@ -87,16 +88,16 @@ func calcShortestPathDistance(center spatialindexer.Location, nearByIDs []*spati return result, nil } -// generateTableRequest generates table requests from center to [startIndex, endIndex] of nearByIDs -func generateTableRequest(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo, startIndex, endIndex int) *table.Request { - if startIndex < 0 || startIndex > endIndex || endIndex >= len(nearByIDs) { - glog.Fatalf("startIndex should be smaller equal to endIndex and both of them should in the range of len(nearByIDs), while (startIndex, endIndex, len(nearByIDs)) = (%d, %d, %d)", - startIndex, endIndex, len(nearByIDs)) +// generateTableRequest generates table requests from center to [startIndex, endIndex] of targets +func generateTableRequest(center spatialindexer.Location, targets []*spatialindexer.PointInfo, startIndex, endIndex int) *table.Request { + if startIndex < 0 || startIndex > endIndex || endIndex >= len(targets) { + glog.Fatalf("startIndex should be smaller equal to endIndex, and both of them should in the range of len(targets), while (startIndex, endIndex, len(targets)) = (%d, %d, %d)", + startIndex, endIndex, len(targets)) } req := table.NewRequest() req.Coordinates = append(convertLocation2Coordinates(center), - convertPointInfos2Coordinates(nearByIDs, startIndex, endIndex)...) + convertPointInfos2Coordinates(targets, startIndex, endIndex)...) req.Sources = append(req.Sources, strconv.Itoa(0)) pointsCount4Sources := 1 @@ -119,12 +120,12 @@ func convertLocation2Coordinates(location spatialindexer.Location) coordinate.Co return result } -func convertPointInfos2Coordinates(nearByIDs []*spatialindexer.PointInfo, startIndex, endIndex int) coordinate.Coordinates { +func convertPointInfos2Coordinates(targets []*spatialindexer.PointInfo, startIndex, endIndex int) coordinate.Coordinates { result := make(coordinate.Coordinates, 0, endIndex-startIndex+1) for i := startIndex; i <= endIndex; i++ { result = append(result, coordinate.Coordinate{ - Lat: nearByIDs[i].Location.Lat, - Lon: nearByIDs[i].Location.Lon, + Lat: targets[i].Location.Lat, + Lon: targets[i].Location.Lon, }) } return result diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go index f2277837e1d..7f2ab783648 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go @@ -1,9 +1,14 @@ package ranker import ( + "encoding/json" + "net/http" + "net/http/httptest" "reflect" + "strings" "testing" + "github.com/Telenav/osrm-backend/integration/oasis/osrmconnector" "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/coordinate" "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/genericoptions" "github.com/Telenav/osrm-backend/integration/pkg/api/osrm/table" @@ -13,18 +18,18 @@ import ( func TestGenerateTableRequest(t *testing.T) { cases := []struct { center spatialindexer.Location - nearByIDs []*spatialindexer.PointInfo + targets []*spatialindexer.PointInfo startIndex int endIndex int expect *table.Request }{ - // case 1: test 0 -> {1, 2, 3, 4, 5} + // case 1: test center -> {1, 2, 3, 4, 5} { center: spatialindexer.Location{ Lat: 0, Lon: 0, }, - nearByIDs: []*spatialindexer.PointInfo{ + targets: []*spatialindexer.PointInfo{ &spatialindexer.PointInfo{ ID: 1, Location: spatialindexer.Location{ @@ -106,13 +111,13 @@ func TestGenerateTableRequest(t *testing.T) { Annotations: "distance,duration", }, }, - // case 2: test 0 -> {2, 3, 4} + // case 2: test center -> {2, 3, 4} { center: spatialindexer.Location{ Lat: 0, Lon: 0, }, - nearByIDs: []*spatialindexer.PointInfo{ + targets: []*spatialindexer.PointInfo{ &spatialindexer.PointInfo{ ID: 1, Location: spatialindexer.Location{ @@ -187,10 +192,190 @@ func TestGenerateTableRequest(t *testing.T) { } for _, c := range cases { - actual := generateTableRequest(c.center, c.nearByIDs, c.startIndex, c.endIndex) + actual := generateTableRequest(c.center, c.targets, c.startIndex, c.endIndex) if !reflect.DeepEqual(actual, c.expect) { t.Errorf("During TestGenerateTableRequest, expect table request is \n%+v\n but actual is \n%+v\n", c.expect, actual) } } } + +func TestRankPointsByOSRMShortestPathWithDifferentPointThresholdPerTableRequest(t *testing.T) { + cases := []struct { + center spatialindexer.Location + targets []*spatialindexer.PointInfo + expect []*spatialindexer.RankedPointInfo + }{ + { + center: spatialindexer.Location{ + Lat: 0, + Lon: 0, + }, + targets: []*spatialindexer.PointInfo{ + &spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 1.1, + Lon: 1.1, + }, + }, + &spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Lat: 2.2, + Lon: 2.2, + }, + }, + &spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 3.3, + Lon: 3.3, + }, + }, + &spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Lat: 4.4, + Lon: 4.4, + }, + }, + &spatialindexer.PointInfo{ + ID: 5, + Location: spatialindexer.Location{ + Lat: 5.5, + Lon: 5.5, + }, + }, + &spatialindexer.PointInfo{ + ID: 6, + Location: spatialindexer.Location{ + Lat: 6.6, + Lon: 6.6, + }, + }, + }, + expect: []*spatialindexer.RankedPointInfo{ + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 1, + Location: spatialindexer.Location{ + Lat: 1.1, + Lon: 1.1, + }, + }, + Distance: 1.1, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 2, + Location: spatialindexer.Location{ + Lat: 2.2, + Lon: 2.2, + }, + }, + Distance: 2.2, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 3, + Location: spatialindexer.Location{ + Lat: 3.3, + Lon: 3.3, + }, + }, + Distance: 3.3, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 4, + Location: spatialindexer.Location{ + Lat: 4.4, + Lon: 4.4, + }, + }, + Distance: 4.4, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 5, + Location: spatialindexer.Location{ + Lat: 5.5, + Lon: 5.5, + }, + }, + Distance: 5.5, + }, + &spatialindexer.RankedPointInfo{ + PointInfo: spatialindexer.PointInfo{ + ID: 6, + Location: spatialindexer.Location{ + Lat: 6.6, + Lon: 6.6, + }, + }, + Distance: 6.6, + }, + }, + }, + } + + // fake OSRM server + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + + if r.Method != "GET" { + t.Errorf("Expected 'GET' request, got '%s'", r.Method) + } + + if strings.HasPrefix(r.URL.EscapedPath(), "/table/v1/driving/") { + req, _ := table.ParseRequestURL(r.URL) + s := len(req.Sources) + d := len(req.Destinations) + if s == 1 && d == 6 { + var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To6TableResponse) + w.Write(tableResponseBytesOrig2Location1) + } else if s == 1 && d == 3 && reflect.DeepEqual(req.Destinations, genericoptions.Elements{"1", "2", "3"}) { + var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To3TableResponsePart1) + w.Write(tableResponseBytesOrig2Location1) + } else if s == 1 && d == 3 && reflect.DeepEqual(req.Destinations, genericoptions.Elements{"4", "5", "6"}) { + var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To3TableResponsePart2) + w.Write(tableResponseBytesOrig2Location1) + } else if s == 1 && d == 4 && reflect.DeepEqual(req.Destinations, genericoptions.Elements{"1", "2", "3", "4"}) { + var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To4TableResponsePart1) + w.Write(tableResponseBytesOrig2Location1) + } else if s == 1 && d == 2 && reflect.DeepEqual(req.Destinations, genericoptions.Elements{"5", "6"}) { + var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To4TableResponsePart2) + w.Write(tableResponseBytesOrig2Location1) + } + return + } + + })) + defer ts.Close() + + oc := osrmconnector.NewOSRMConnector(ts.URL) + + for _, c := range cases { + actual := rankPointsByOSRMShortestPath(c.center, c.targets, oc, 999) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("During TestRankerInterfaceViaOSRMRanker, expect \n%s \nwhile actual result is \n%s\n", + printRankedPointInfoArray(c.expect), + printRankedPointInfoArray(actual)) + } + + actual = rankPointsByOSRMShortestPath(c.center, c.targets, oc, 3) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("During TestRankerInterfaceViaOSRMRanker, expect \n%s \nwhile actual result is \n%s\n", + printRankedPointInfoArray(c.expect), + printRankedPointInfoArray(actual)) + } + + actual = rankPointsByOSRMShortestPath(c.center, c.targets, oc, 4) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("During TestRankerInterfaceViaOSRMRanker, expect \n%s \nwhile actual result is \n%s\n", + printRankedPointInfoArray(c.expect), + printRankedPointInfoArray(actual)) + } + } +} diff --git a/integration/service/spatialindexer/ranker/simple_ranker.go b/integration/service/spatialindexer/ranker/simple_ranker.go index c765ce9f29e..c675cad8d90 100644 --- a/integration/service/spatialindexer/ranker/simple_ranker.go +++ b/integration/service/spatialindexer/ranker/simple_ranker.go @@ -9,10 +9,10 @@ func newSimpleRanker() *simpleRanker { return &simpleRanker{} } -func (ranker *simpleRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { - return rankPointsByGreatCircleDistanceToCenter(center, nearByIDs) +func (ranker *simpleRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + return rankPointsByGreatCircleDistanceToCenter(center, targets) } -func (ranker *simpleRanker) RankPointIDsByShortestDistance(center spatialindexer.Location, nearByIDs []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { - return ranker.RankPointIDsByGreatCircleDistance(center, nearByIDs) +func (ranker *simpleRanker) RankPointIDsByShortestDistance(center spatialindexer.Location, targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { + return ranker.RankPointIDsByGreatCircleDistance(center, targets) } diff --git a/integration/service/spatialindexer/ranker/simple_ranker_test.go b/integration/service/spatialindexer/ranker/simple_ranker_test.go index 1dce8bcdd2c..e7554deb569 100644 --- a/integration/service/spatialindexer/ranker/simple_ranker_test.go +++ b/integration/service/spatialindexer/ranker/simple_ranker_test.go @@ -9,16 +9,16 @@ import ( func TestRankerInterfaceViaSimpleRanker(t *testing.T) { cases := []struct { - center spatialindexer.Location - nearByIDs []*spatialindexer.PointInfo - expect []*spatialindexer.RankedPointInfo + center spatialindexer.Location + targets []*spatialindexer.PointInfo + expect []*spatialindexer.RankedPointInfo }{ { center: spatialindexer.Location{ Lat: 37.398973, Lon: -121.976633, }, - nearByIDs: []*spatialindexer.PointInfo{ + targets: []*spatialindexer.PointInfo{ &spatialindexer.PointInfo{ ID: 1, Location: spatialindexer.Location{ @@ -78,14 +78,14 @@ func TestRankerInterfaceViaSimpleRanker(t *testing.T) { ranker := CreateRanker(SimpleRanker, nil) for _, c := range cases { - actual := ranker.RankPointIDsByGreatCircleDistance(c.center, c.nearByIDs) + actual := ranker.RankPointIDsByGreatCircleDistance(c.center, c.targets) if !reflect.DeepEqual(actual, c.expect) { t.Errorf("During test SimpleRanker's RankPointIDsByGreatCircleDistance, \n expect \n%s \nwhile actual is\n %s\n", printRankedPointInfoArray(c.expect), printRankedPointInfoArray(actual)) } - actual = ranker.RankPointIDsByGreatCircleDistance(c.center, c.nearByIDs) + actual = ranker.RankPointIDsByShortestDistance(c.center, c.targets) if !reflect.DeepEqual(actual, c.expect) { t.Errorf("During test SimpleRanker's RankPointIDsByGreatCircleDistance, \n expect \n%s \nwhile actual is\n %s\n", printRankedPointInfoArray(c.expect), From 8fa567ea2210f97843352ef00732c8431c986baf Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 31 Mar 2020 10:21:15 -0700 Subject: [PATCH 15/27] fix: update code based on golang-lint's suggestions. --- integration/service/spatialindexer/ranker/doc.go | 4 +++- .../service/spatialindexer/ranker/rank_agent.go | 1 + .../spatialindexer/ranker/rank_agent_test.go | 2 +- .../ranker/rank_by_osrm_shortest_path_impl.go | 13 ++++++++----- .../ranker/rank_by_osrm_shortest_path_impl_test.go | 2 +- .../service/spatialindexer/ranker/simple_ranker.go | 6 ++++-- .../spatialindexer/ranker/simple_ranker_test.go | 1 + 7 files changed, 19 insertions(+), 10 deletions(-) diff --git a/integration/service/spatialindexer/ranker/doc.go b/integration/service/spatialindexer/ranker/doc.go index d1ca5c5439c..0b1853202a9 100644 --- a/integration/service/spatialindexer/ranker/doc.go +++ b/integration/service/spatialindexer/ranker/doc.go @@ -1,4 +1,6 @@ /* -Package ranker implements interfaces defined in Ranker. +Package ranker implements interfaces defined in Ranker: + - RankPointIDsByGreatCircleDistance + - RankPointIDsByShortestDistance */ package ranker diff --git a/integration/service/spatialindexer/ranker/rank_agent.go b/integration/service/spatialindexer/ranker/rank_agent.go index a121a6d5ad7..ef8c7ec647b 100644 --- a/integration/service/spatialindexer/ranker/rank_agent.go +++ b/integration/service/spatialindexer/ranker/rank_agent.go @@ -6,6 +6,7 @@ import ( "github.com/Telenav/osrm-backend/integration/service/spatialindexer" ) +// rankAgent accepts items to be ranked then returns ranking result type rankAgent struct { rankedPoints []*spatialindexer.RankedPointInfo } diff --git a/integration/service/spatialindexer/ranker/rank_agent_test.go b/integration/service/spatialindexer/ranker/rank_agent_test.go index 8e5a951c351..ee1ac9fa4a1 100644 --- a/integration/service/spatialindexer/ranker/rank_agent_test.go +++ b/integration/service/spatialindexer/ranker/rank_agent_test.go @@ -105,8 +105,8 @@ func TestRankAgent(t *testing.T) { for _, c := range cases { var wg sync.WaitGroup pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(c.input)) + wg.Add(1) go func(wg *sync.WaitGroup) { - wg.Add(1) defer wg.Done() for _, item := range c.input { diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go index 1fd6ae64fc5..5988c6a5cf2 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go @@ -13,12 +13,13 @@ import ( "github.com/golang/glog" ) -// pointsThresholdPerRequest defines point count seperator for each single table request. +// pointsThresholdPerRequest defines point count separator for each single table request. // During pre-processing, its possible to have situation to calculate distance between thousnads of points. // The situation here is 1-to-N table request, use pointsLimit4SingleTableRequest to limit N const pointsThresholdPerRequest = 1000 -func rankPointsByOSRMShortestPath(center spatialindexer.Location, targets []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector, pointsThreshold int) []*spatialindexer.RankedPointInfo { +func rankPointsByOSRMShortestPath(center spatialindexer.Location, targets []*spatialindexer.PointInfo, + oc *osrmconnector.OSRMConnector, pointsThreshold int) []*spatialindexer.RankedPointInfo { if len(targets) == 0 { glog.Warning("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty\n") return nil @@ -40,10 +41,12 @@ func rankPointsByOSRMShortestPath(center spatialindexer.Location, targets []*spa wg.Add(1) go func(wg *sync.WaitGroup, startIndex, endIndex int) { - rankedPoints, err := calcShortestPathDistance(center, targets, oc, startIndex, endIndex) + rankedPoints, err := calcCenter2TargetsDistanceViaShortestPath(center, targets, oc, startIndex, endIndex) if err != nil { - // @todo: add retry logic when failed + glog.Errorf("Failed to calculate shortest path for range [%d, %d] for center = %+v, targets = %+v\n", + startIndex, endIndex, center, targets) + // @todo: add retry logic when failed or may be put retry logic in connector } else { for _, item := range rankedPoints { pointWithDistanceC <- item @@ -64,7 +67,7 @@ func rankPointsByOSRMShortestPath(center spatialindexer.Location, targets []*spa } -func calcShortestPathDistance(center spatialindexer.Location, targets []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector, startIndex, endIndex int) ([]*spatialindexer.RankedPointInfo, error) { +func calcCenter2TargetsDistanceViaShortestPath(center spatialindexer.Location, targets []*spatialindexer.PointInfo, oc *osrmconnector.OSRMConnector, startIndex, endIndex int) ([]*spatialindexer.RankedPointInfo, error) { req := generateTableRequest(center, targets, startIndex, endIndex) respC := oc.Request4Table(req) resp := <-respC diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go index 7f2ab783648..2804ef7fa1f 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go @@ -357,7 +357,7 @@ func TestRankPointsByOSRMShortestPathWithDifferentPointThresholdPerTableRequest( oc := osrmconnector.NewOSRMConnector(ts.URL) for _, c := range cases { - actual := rankPointsByOSRMShortestPath(c.center, c.targets, oc, 999) + actual := rankPointsByOSRMShortestPath(c.center, c.targets, oc, pointsThresholdPerRequest) if !reflect.DeepEqual(actual, c.expect) { t.Errorf("During TestRankerInterfaceViaOSRMRanker, expect \n%s \nwhile actual result is \n%s\n", printRankedPointInfoArray(c.expect), diff --git a/integration/service/spatialindexer/ranker/simple_ranker.go b/integration/service/spatialindexer/ranker/simple_ranker.go index c675cad8d90..09c8e9c5211 100644 --- a/integration/service/spatialindexer/ranker/simple_ranker.go +++ b/integration/service/spatialindexer/ranker/simple_ranker.go @@ -9,10 +9,12 @@ func newSimpleRanker() *simpleRanker { return &simpleRanker{} } -func (ranker *simpleRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { +func (ranker *simpleRanker) RankPointIDsByGreatCircleDistance(center spatialindexer.Location, + targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { return rankPointsByGreatCircleDistanceToCenter(center, targets) } -func (ranker *simpleRanker) RankPointIDsByShortestDistance(center spatialindexer.Location, targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { +func (ranker *simpleRanker) RankPointIDsByShortestDistance(center spatialindexer.Location, + targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo { return ranker.RankPointIDsByGreatCircleDistance(center, targets) } diff --git a/integration/service/spatialindexer/ranker/simple_ranker_test.go b/integration/service/spatialindexer/ranker/simple_ranker_test.go index e7554deb569..a54d1b07882 100644 --- a/integration/service/spatialindexer/ranker/simple_ranker_test.go +++ b/integration/service/spatialindexer/ranker/simple_ranker_test.go @@ -77,6 +77,7 @@ func TestRankerInterfaceViaSimpleRanker(t *testing.T) { } ranker := CreateRanker(SimpleRanker, nil) + for _, c := range cases { actual := ranker.RankPointIDsByGreatCircleDistance(c.center, c.targets) if !reflect.DeepEqual(actual, c.expect) { From 1abfe70837bda74e19a3f3558a6c3969c8d1b543 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 31 Mar 2020 10:39:40 -0700 Subject: [PATCH 16/27] fix: adjust comments --- .../spatialindexer/ranker/osrm_ranker_test.go | 186 +++++++++--------- .../ranker/rank_by_osrm_shortest_path_impl.go | 11 +- .../rank_by_osrm_shortest_path_impl_test.go | 9 +- 3 files changed, 107 insertions(+), 99 deletions(-) diff --git a/integration/service/spatialindexer/ranker/osrm_ranker_test.go b/integration/service/spatialindexer/ranker/osrm_ranker_test.go index 802e515ff83..77ad0bf3791 100644 --- a/integration/service/spatialindexer/ranker/osrm_ranker_test.go +++ b/integration/service/spatialindexer/ranker/osrm_ranker_test.go @@ -13,99 +13,6 @@ import ( "github.com/Telenav/osrm-backend/integration/service/spatialindexer" ) -var mockFloatArray1To6 []float64 = []float64{1.1, 2.2, 3.3, 4.4, 5.5, 6.6} - -var mock1To6TableResponse table.Response = table.Response{ - Durations: [][]*float64{ - []*float64{ - &mockFloatArray1To6[0], - &mockFloatArray1To6[1], - &mockFloatArray1To6[2], - &mockFloatArray1To6[3], - &mockFloatArray1To6[4], - &mockFloatArray1To6[5], - }, - }, - Distances: [][]*float64{ - []*float64{ - &mockFloatArray1To6[0], - &mockFloatArray1To6[1], - &mockFloatArray1To6[2], - &mockFloatArray1To6[3], - &mockFloatArray1To6[4], - &mockFloatArray1To6[5], - }, - }, -} - -var mock1To3TableResponsePart1 table.Response = table.Response{ - Durations: [][]*float64{ - []*float64{ - &mockFloatArray1To6[0], - &mockFloatArray1To6[1], - &mockFloatArray1To6[2], - }, - }, - Distances: [][]*float64{ - []*float64{ - &mockFloatArray1To6[0], - &mockFloatArray1To6[1], - &mockFloatArray1To6[2], - }, - }, -} - -var mock1To3TableResponsePart2 table.Response = table.Response{ - Durations: [][]*float64{ - []*float64{ - &mockFloatArray1To6[3], - &mockFloatArray1To6[4], - &mockFloatArray1To6[5], - }, - }, - Distances: [][]*float64{ - []*float64{ - &mockFloatArray1To6[3], - &mockFloatArray1To6[4], - &mockFloatArray1To6[5], - }, - }, -} - -var mock1To4TableResponsePart1 table.Response = table.Response{ - Durations: [][]*float64{ - []*float64{ - &mockFloatArray1To6[0], - &mockFloatArray1To6[1], - &mockFloatArray1To6[2], - &mockFloatArray1To6[3], - }, - }, - Distances: [][]*float64{ - []*float64{ - &mockFloatArray1To6[0], - &mockFloatArray1To6[1], - &mockFloatArray1To6[2], - &mockFloatArray1To6[3], - }, - }, -} - -var mock1To4TableResponsePart2 table.Response = table.Response{ - Durations: [][]*float64{ - []*float64{ - &mockFloatArray1To6[4], - &mockFloatArray1To6[5], - }, - }, - Distances: [][]*float64{ - []*float64{ - &mockFloatArray1To6[4], - &mockFloatArray1To6[5], - }, - }, -} - func TestRankerInterfaceViaOSRMRanker(t *testing.T) { cases := []struct { center spatialindexer.Location @@ -260,3 +167,96 @@ func TestRankerInterfaceViaOSRMRanker(t *testing.T) { } } } + +var mockFloatArray1To6 []float64 = []float64{1.1, 2.2, 3.3, 4.4, 5.5, 6.6} + +var mock1To6TableResponse table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + &mockFloatArray1To6[3], + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + &mockFloatArray1To6[3], + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, +} + +var mock1To3TableResponsePart1 table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + }, + }, +} + +var mock1To3TableResponsePart2 table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[3], + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[3], + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, +} + +var mock1To4TableResponsePart1 table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + &mockFloatArray1To6[3], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[0], + &mockFloatArray1To6[1], + &mockFloatArray1To6[2], + &mockFloatArray1To6[3], + }, + }, +} + +var mock1To4TableResponsePart2 table.Response = table.Response{ + Durations: [][]*float64{ + []*float64{ + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, + Distances: [][]*float64{ + []*float64{ + &mockFloatArray1To6[4], + &mockFloatArray1To6[5], + }, + }, +} diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go index 5988c6a5cf2..5d48a4fa5fb 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl.go @@ -13,9 +13,10 @@ import ( "github.com/golang/glog" ) -// pointsThresholdPerRequest defines point count separator for each single table request. -// During pre-processing, its possible to have situation to calculate distance between thousnads of points. -// The situation here is 1-to-N table request, use pointsLimit4SingleTableRequest to limit N +// pointsThresholdPerRequest limits max point count for each single table request. +// During pre-processing, its possible to calculate distance between thousands of points, which will +// cause too big request and might reach potential limitation of different parts. +// The scenario here is 1-to-N table request, use pointsLimit4SingleTableRequest to limit N const pointsThresholdPerRequest = 1000 func rankPointsByOSRMShortestPath(center spatialindexer.Location, targets []*spatialindexer.PointInfo, @@ -27,8 +28,8 @@ func rankPointsByOSRMShortestPath(center spatialindexer.Location, targets []*spa var wg sync.WaitGroup pointWithDistanceC := make(chan *spatialindexer.RankedPointInfo, len(targets)) - startIndex := 0 - endIndex := 0 + startIndex := 0 // startIndex is a valid index of targets + endIndex := 0 // endIndex is valid index of targets for { if startIndex >= len(targets) { break diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go index 2804ef7fa1f..c8e0baf901c 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go @@ -200,7 +200,14 @@ func TestGenerateTableRequest(t *testing.T) { } -func TestRankPointsByOSRMShortestPathWithDifferentPointThresholdPerTableRequest(t *testing.T) { +// TestRankPointsByOSRMShortestPathWithDifferentPointThreshold tries with different threshold and assert for same response +// In this case, target array contains 6 points {1, 2, 3, 4, 5, 6} +// when pointThreshold = pointsThresholdPerRequest(MaxInt32), all of them will be put in the same request +// when pointThreshold = 3, will divide points into two request, {1, 2, 3}, {4, 5, 6} +// but the result will be merged and sorted, so is the same with single request +// when pointThreshold = 4, will divide points into two request, {1, 2, 3, 4}, {5, 6} +// but the result will be merged and sorted, so is the same with single request +func TestRankPointsByOSRMShortestPathWithDifferentPointThreshold(t *testing.T) { cases := []struct { center spatialindexer.Location targets []*spatialindexer.PointInfo From 7c8815b364956429ecf283d18b2caf408899f813 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 31 Mar 2020 10:42:16 -0700 Subject: [PATCH 17/27] fix: remove placeconnectivitymap and will keep it for next commit. --- .../placeconnectivitymap/connectivity_map.go | 27 ---------------- .../service/placeconnectivitymap/doc.go | 32 ------------------- 2 files changed, 59 deletions(-) delete mode 100644 integration/service/placeconnectivitymap/connectivity_map.go delete mode 100644 integration/service/placeconnectivitymap/doc.go diff --git a/integration/service/placeconnectivitymap/connectivity_map.go b/integration/service/placeconnectivitymap/connectivity_map.go deleted file mode 100644 index 4bd4b51337f..00000000000 --- a/integration/service/placeconnectivitymap/connectivity_map.go +++ /dev/null @@ -1,27 +0,0 @@ -package placeconnectivitymap - -import ( - "github.com/Telenav/osrm-backend/integration/service/spatialindexer" - "github.com/Telenav/osrm-backend/integration/service/spatialindexer/s2indexer" - "github.com/golang/glog" -) - -type PlaceConnectivityMap struct { - finder spatialindexer.Finder - ranker spatialindexer.Ranker - iterator spatialindexer.PointsIterator -} - -func NewPlaceConnectivityMap(filePath string) *PlaceConnectivityMap { - indexer := s2indexer.NewS2Indexer().Build(filePath) - if indexer == nil { - glog.Error("Failed to NewPlaceConnectivityMap due to empty indexer is generated. Check your input\n") - return nil - } - - return nil -} - -func (pcm *PlaceConnectivityMap) GenerateConnectivityMap(filePath string) { - glog.Info("Successfully finished GenerateConnectivityMap\n") -} diff --git a/integration/service/placeconnectivitymap/doc.go b/integration/service/placeconnectivitymap/doc.go deleted file mode 100644 index a9fa85a83c9..00000000000 --- a/integration/service/placeconnectivitymap/doc.go +++ /dev/null @@ -1,32 +0,0 @@ -/* -package placeconnectivitymap provides connectivity information for place graphs. - -For example, given following graph - - - - - - 4 -| | | | | - - 2 - - - -| | | | | - - - - 3 - -| | | | | -1 - - - - - -Place connectivity map will pre-process each points and generate following result: -While query connectivity for place 1, will return - (place 2, 3), // the shortest path between place 1 and place 2 is 3 - (place 3, 4), // the shortest path between place 1 and place 2 is 4 - (place 4, 7), // the shortest path between place 1 and place 2 is 7 -The result is sorted by shortest path distance(or other user defined strategy) - -When query for connectivity, user could also pass in limitation option, such as distance limitation. -For example, when query connectivity for place 3 -With limitation = -1, it will return - (place 2, 3), // the shortest path between place 3 and place 2 is 3 - (place 4, 3), // the shortest path between place 3 and place 4 is 3 - (place 1, 4), // the shortest path between place 3 and place 1 is 4 -With limitation = 3, it will return - (place 2, 3), // the shortest path between place 3 and place 2 is 3 - (place 4, 3), // the shortest path between place 3 and place 4 is 3 - -*/ -package placeconnectivitymap From 367138ad386b2371ed3dd7aeb1a5906cf1729697 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 31 Mar 2020 10:55:21 -0700 Subject: [PATCH 18/27] fix: adjust names --- .../rank_by_osrm_shortest_path_impl_test.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go index c8e0baf901c..fb6df354709 100644 --- a/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go +++ b/integration/service/spatialindexer/ranker/rank_by_osrm_shortest_path_impl_test.go @@ -340,20 +340,20 @@ func TestRankPointsByOSRMShortestPathWithDifferentPointThreshold(t *testing.T) { s := len(req.Sources) d := len(req.Destinations) if s == 1 && d == 6 { - var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To6TableResponse) - w.Write(tableResponseBytesOrig2Location1) + var tableResponseBytes, _ = json.Marshal(mock1To6TableResponse) + w.Write(tableResponseBytes) } else if s == 1 && d == 3 && reflect.DeepEqual(req.Destinations, genericoptions.Elements{"1", "2", "3"}) { - var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To3TableResponsePart1) - w.Write(tableResponseBytesOrig2Location1) + var tableResponseBytes, _ = json.Marshal(mock1To3TableResponsePart1) + w.Write(tableResponseBytes) } else if s == 1 && d == 3 && reflect.DeepEqual(req.Destinations, genericoptions.Elements{"4", "5", "6"}) { - var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To3TableResponsePart2) - w.Write(tableResponseBytesOrig2Location1) + var tableResponseBytes, _ = json.Marshal(mock1To3TableResponsePart2) + w.Write(tableResponseBytes) } else if s == 1 && d == 4 && reflect.DeepEqual(req.Destinations, genericoptions.Elements{"1", "2", "3", "4"}) { - var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To4TableResponsePart1) - w.Write(tableResponseBytesOrig2Location1) + var tableResponseBytes, _ = json.Marshal(mock1To4TableResponsePart1) + w.Write(tableResponseBytes) } else if s == 1 && d == 2 && reflect.DeepEqual(req.Destinations, genericoptions.Elements{"5", "6"}) { - var tableResponseBytesOrig2Location1, _ = json.Marshal(mock1To4TableResponsePart2) - w.Write(tableResponseBytesOrig2Location1) + var tableResponseBytes, _ = json.Marshal(mock1To4TableResponsePart2) + w.Write(tableResponseBytes) } return } From 05261bafc5a1dd15810cbec13a8d1fd7505a005d Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 31 Mar 2020 11:05:00 -0700 Subject: [PATCH 19/27] fix: update comment. --- integration/service/spatialindexer/interface.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/service/spatialindexer/interface.go b/integration/service/spatialindexer/interface.go index ef347245135..1f4b885c9e7 100644 --- a/integration/service/spatialindexer/interface.go +++ b/integration/service/spatialindexer/interface.go @@ -25,7 +25,7 @@ type RankedPointInfo struct { // Only the data used for pre-processing contains valid PointID type PointID int64 -// UnlimitedCount indicate all spatial search result will be returned +// UnlimitedCount means all spatial search result will be returned const UnlimitedCount = math.MaxInt32 // Finder answers special query From f4308538896069105fe6509081b8c181689834e1 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Tue, 31 Mar 2020 18:13:12 -0700 Subject: [PATCH 20/27] feat: implement connectivitymap. issue: https://github.com/Telenav/osrm-backend/issues/238 --- .../service/connectivitymap/builder.go | 65 ++++++++ .../connectivitymap/connectivity_map.go | 46 ++++++ .../connectivitymap/connectivity_map_mock.go | 66 ++++++++ integration/service/connectivitymap/doc.go | 32 ++++ integration/service/connectivitymap/dumper.go | 118 ++++++++++++++ .../service/connectivitymap/dumper_test.go | 60 +++++++ .../service/connectivitymap/statistic.go | 152 ++++++++++++++++++ .../service/connectivitymap/statistic_test.go | 41 +++++ .../service/spatialindexer/interface.go | 2 +- .../spatialindexer/s2indexer/indexer.go | 2 +- .../spatialindexer/s2indexer/spatial_query.go | 6 +- .../s2indexer/spatial_query_test.go | 6 +- 12 files changed, 588 insertions(+), 8 deletions(-) create mode 100644 integration/service/connectivitymap/builder.go create mode 100644 integration/service/connectivitymap/connectivity_map.go create mode 100644 integration/service/connectivitymap/connectivity_map_mock.go create mode 100644 integration/service/connectivitymap/doc.go create mode 100644 integration/service/connectivitymap/dumper.go create mode 100644 integration/service/connectivitymap/dumper_test.go create mode 100644 integration/service/connectivitymap/statistic.go create mode 100644 integration/service/connectivitymap/statistic_test.go diff --git a/integration/service/connectivitymap/builder.go b/integration/service/connectivitymap/builder.go new file mode 100644 index 00000000000..b0ce5594d19 --- /dev/null +++ b/integration/service/connectivitymap/builder.go @@ -0,0 +1,65 @@ +package connectivitymap + +import "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + +type connectivityMapBuilder struct { + iterator spatialindexer.PointsIterator + finder spatialindexer.Finder + ranker spatialindexer.Ranker + distanceLimit float64 +} + +func newConnectivityMapBuilder(iterator spatialindexer.PointsIterator, finder spatialindexer.Finder, + ranker spatialindexer.Ranker, distanceLimit float64) *connectivityMapBuilder { + return &connectivityMapBuilder{ + iterator: iterator, + finder: finder, + ranker: ranker, + distanceLimit: distanceLimit, + } +} + +type placeIDWithNearByPlaceIDs struct { + id spatialindexer.PointID + ids []IDAndDistance +} + +// todo: use task pool +func (builder *connectivityMapBuilder) build() ID2NearByIDsMap { + internalResult := make(chan placeIDWithNearByPlaceIDs, 10000) + m := make(ID2NearByIDsMap) + + go func() { + for p := range builder.iterator.IteratePoints() { + nearbyIDs := builder.finder.FindNearByPointIDs(p.Location, builder.distanceLimit, spatialindexer.UnlimitedCount) + rankedResults := builder.ranker.RankPointIDsByGreatCircleDistance(p.Location, nearbyIDs) + + ids := make([]IDAndDistance, 0, len(rankedResults)) + for _, r := range rankedResults { + ids = append(ids, IDAndDistance{ + ID: r.ID, + Distance: r.Distance, + }) + } + internalResult <- placeIDWithNearByPlaceIDs{ + id: p.ID, + ids: ids, + } + } + close(internalResult) + }() + + for item := range internalResult { + m[item.id] = item.ids + } + + return m +} + +// func (builder *connectivityMapBuilder) dump(folderPath string, m ID2NearByIDsMap) { + +// } + +// func (builder *connectivityMapBuilder) load(folderPath string) (ID2NearByIDsMap, float64) { + +// } diff --git a/integration/service/connectivitymap/connectivity_map.go b/integration/service/connectivitymap/connectivity_map.go new file mode 100644 index 00000000000..9d68294ce9c --- /dev/null +++ b/integration/service/connectivitymap/connectivity_map.go @@ -0,0 +1,46 @@ +package connectivitymap + +import ( + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/glog" +) + +// IDAndDistance wraps ID and distance information +type IDAndDistance struct { + ID spatialindexer.PointID + Distance float64 +} + +// ID2NearByIDsMap is a mapping between ID and its nearby IDs +type ID2NearByIDsMap map[spatialindexer.PointID][]IDAndDistance + +type ConnectivityMap struct { + id2nearByIDs ID2NearByIDsMap + distanceLimitation float64 + statistic *statistic +} + +func (cm *ConnectivityMap) DistanceLimitation() float64 { + return cm.distanceLimitation +} + +func NewPlaceConnectivityMap(distanceLimitation float64) *ConnectivityMap { + return &ConnectivityMap{ + distanceLimitation: distanceLimitation, + statistic: newStatistic(), + } +} + +func (cm *ConnectivityMap) Build() { + glog.Info("Successfully finished GenerateConnectivityMap\n") +} + +func (cm *ConnectivityMap) Dump(folderPath string) { +} + +func (cm *ConnectivityMap) Load(folderPath string) { +} + +func (cm *ConnectivityMap) QueryConnectivity(placeInfo spatialindexer.PointInfo, limitDistance float64) { + // for each everything recorded in data, apply limit option on that +} diff --git a/integration/service/connectivitymap/connectivity_map_mock.go b/integration/service/connectivitymap/connectivity_map_mock.go new file mode 100644 index 00000000000..0bea5c8cb3e --- /dev/null +++ b/integration/service/connectivitymap/connectivity_map_mock.go @@ -0,0 +1,66 @@ +package connectivitymap + +var fakeID2NearByIDsMap1 = ID2NearByIDsMap{ + 1: []IDAndDistance{ + IDAndDistance{ + ID: 2, + Distance: 3, + }, + IDAndDistance{ + ID: 5, + Distance: 4, + }, + IDAndDistance{ + ID: 7, + Distance: 6, + }, + IDAndDistance{ + ID: 8, + Distance: 12, + }, + }, + + 2: []IDAndDistance{ + IDAndDistance{ + ID: 1, + Distance: 3, + }, + IDAndDistance{ + ID: 7, + Distance: 23, + }, + }, + + 5: []IDAndDistance{ + IDAndDistance{ + ID: 1, + Distance: 4, + }, + IDAndDistance{ + ID: 8, + Distance: 5, + }, + }, + + 7: []IDAndDistance{ + IDAndDistance{ + ID: 1, + Distance: 6, + }, + IDAndDistance{ + ID: 2, + Distance: 23, + }, + }, + + 8: []IDAndDistance{ + IDAndDistance{ + ID: 5, + Distance: 5, + }, + IDAndDistance{ + ID: 1, + Distance: 12, + }, + }, +} diff --git a/integration/service/connectivitymap/doc.go b/integration/service/connectivitymap/doc.go new file mode 100644 index 00000000000..a6237bd377e --- /dev/null +++ b/integration/service/connectivitymap/doc.go @@ -0,0 +1,32 @@ +/* +package placeconnectivitymap provides connectivity information for place graphs. + +For example, given following graph + + - - - - 4 +| | | | | + - 2 - - - +| | | | | + - - - 3 - +| | | | | +1 - - - - + +Place connectivity map will pre-process each points and generate following result: +While query connectivity for place 1, will return + (place 2, 3), // the shortest path between place 1 and place 2 is 3 + (place 3, 4), // the shortest path between place 1 and place 2 is 4 + (place 4, 7), // the shortest path between place 1 and place 2 is 7 +The result is sorted by shortest path distance(or other user defined strategy) + +When query for connectivity, user could also pass in limitation option, such as distance limitation. +For example, when query connectivity for place 3 +With limitation = -1, it will return + (place 2, 3), // the shortest path between place 3 and place 2 is 3 + (place 4, 3), // the shortest path between place 3 and place 4 is 3 + (place 1, 4), // the shortest path between place 3 and place 1 is 4 +With limitation = 3, it will return + (place 2, 3), // the shortest path between place 3 and place 2 is 3 + (place 4, 3), // the shortest path between place 3 and place 4 is 3 + +*/ +package connectivitymap diff --git a/integration/service/connectivitymap/dumper.go b/integration/service/connectivitymap/dumper.go new file mode 100644 index 00000000000..b8d21441f51 --- /dev/null +++ b/integration/service/connectivitymap/dumper.go @@ -0,0 +1,118 @@ +package connectivitymap + +import ( + "bytes" + "encoding/gob" + "io/ioutil" + "os" + "strings" + + "github.com/Telenav/osrm-backend/integration/pkg/api" + "github.com/golang/glog" +) + +const id2NearByIDsMapFileName = "id2nearbyidsmap.gob" + +func serializeConnectivityMap(cm *ConnectivityMap, folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath += api.Slash + } + + if err := serializeID2NearByIDsMap(cm, folderPath); err != nil { + return err + } + + if err := cm.statistic.dump(folderPath); err != nil { + return err + } + + glog.Infof("Successfully deserialize ConnectivityMap from folder %s.\n", folderPath) + + return nil +} + +func deSerializeConnectivityMap(cm *ConnectivityMap, folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath += api.Slash + } + + if err := deSerializeID2NearByIDsMap(cm, folderPath); err != nil { + return err + } + + if err := cm.statistic.load(folderPath); err != nil { + return err + } + cm.distanceLimitation = cm.statistic.DistanceLimitation + + return nil +} + +func removeAllDumpFiles(folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath += "/" + } + + _, err := os.Stat(folderPath + id2NearByIDsMapFileName) + if !os.IsNotExist(err) { + err = os.Remove(folderPath + id2NearByIDsMapFileName) + if err != nil { + glog.Errorf("Remove file failed %s\n", folderPath+id2NearByIDsMapFileName) + return err + } + } else { + glog.Warningf("There is no %s file in folder %s\n", id2NearByIDsMapFileName, folderPath) + } + + _, err = os.Stat(folderPath + statisticFileName) + if !os.IsNotExist(err) { + err = os.Remove(folderPath + statisticFileName) + if err != nil { + glog.Errorf("Remove file failed %s\n", folderPath+statisticFileName) + return err + } + } else { + glog.Warningf("There is no %s file in folder %s\n", statisticFileName, folderPath) + } + + return nil +} + +func serializeID2NearByIDsMap(cm *ConnectivityMap, folderPath string) error { + buf := new(bytes.Buffer) + encoder := gob.NewEncoder(buf) + err := encoder.Encode(cm.id2nearByIDs) + + if err != nil { + glog.Errorf("During encode ConnectivityMap's ID2NearByIDsMap met error %v", err) + return err + } + + if err = ioutil.WriteFile(folderPath+id2NearByIDsMapFileName, buf.Bytes(), 0644); err != nil { + glog.Errorf("During dump ConnectivityMap's ID2NearByIDsMap to %s met error %v", folderPath+id2NearByIDsMapFileName, err) + return err + } + + return nil +} + +func deSerializeID2NearByIDsMap(cm *ConnectivityMap, folderPath string) error { + byteArray, err := ioutil.ReadFile(folderPath + id2NearByIDsMapFileName) + if err != nil { + glog.Errorf("During load ConnectivityMap's ID2NearByIDsMap from %s met error %v", + folderPath+id2NearByIDsMapFileName, err) + return err + } + + buf := bytes.NewBuffer(byteArray) + decoder := gob.NewDecoder(buf) + err = decoder.Decode(&cm.id2nearByIDs) + + if err != nil { + glog.Errorf("During decode ConnectivityMap's ID2NearByIDsMap from %s met error %v", + folderPath+id2NearByIDsMapFileName, err) + return err + } + + return nil +} diff --git a/integration/service/connectivitymap/dumper_test.go b/integration/service/connectivitymap/dumper_test.go new file mode 100644 index 00000000000..ec94f16fa6c --- /dev/null +++ b/integration/service/connectivitymap/dumper_test.go @@ -0,0 +1,60 @@ +package connectivitymap + +import ( + "os" + "reflect" + "testing" + + "github.com/golang/glog" +) + +func TestDumper(t *testing.T) { + cases := []ConnectivityMap{ + ConnectivityMap{ + id2nearByIDs: fakeID2NearByIDsMap1, + distanceLimitation: fakeDistanceLimit, + statistic: &fakeStatisticResult1, + }, + } + + // check whether curent folder is writeable + path, _ := os.Getwd() + _, err := os.OpenFile(path+"/tmp", os.O_RDONLY|os.O_CREATE, 0644) + if err != nil { + return + } + if err := os.Remove(path + "/tmp"); err != nil { + glog.Errorf("During dumper test, remove path %s failed.\n", path+"/tmp") + return + } + + if err := removeAllDumpFiles(path); err != nil { + t.Errorf("During running removeAllDumpFiles met error %v", err) + } + for _, c := range cases { + if err := serializeConnectivityMap(&c, path); err != nil { + t.Errorf("During running serializeConnectivityMap for case %v, met error %v", c, err) + } + + actual := NewPlaceConnectivityMap(0.0) + if err := deSerializeConnectivityMap(actual, path); err != nil { + t.Errorf("During running deSerializeConnectivityMap for case %v, met error %v", c, err) + } + + if !reflect.DeepEqual(actual.id2nearByIDs, c.id2nearByIDs) { + t.Errorf("Expect result \n%+v but got \n%+v\n", c.id2nearByIDs, actual.id2nearByIDs) + } + + if !reflect.DeepEqual(actual.distanceLimitation, c.distanceLimitation) { + t.Errorf("Expect result \n%+v but got \n%+v\n", c.distanceLimitation, actual.distanceLimitation) + } + + if !reflect.DeepEqual(actual.statistic, c.statistic) { + t.Errorf("Expect result \n%+v but got \n%+v\n", c.statistic, actual.statistic) + } + + if err := removeAllDumpFiles(path); err != nil { + t.Errorf("During running removeAllDumpFiles met error %v", err) + } + } +} diff --git a/integration/service/connectivitymap/statistic.go b/integration/service/connectivitymap/statistic.go new file mode 100644 index 00000000000..5df25774f4d --- /dev/null +++ b/integration/service/connectivitymap/statistic.go @@ -0,0 +1,152 @@ +package connectivitymap + +import ( + "encoding/json" + "io/ioutil" + "math" + "strings" + + "github.com/Telenav/osrm-backend/integration/pkg/api" + "github.com/golang/glog" +) + +const statisticFileName = "connectivity_map_statistic.json" + +type statistic struct { + Count int `json:"count"` + ValidCount int `json:"valid_count"` + AverageNearByIDsCount int `json:"average_nearby_ids_count"` + MaxNearByIDsCount int `json:"max_nearby_ids_count"` + MinNearByIDsCount int `json:"min_nearby_ids_count"` + AverageMaxDistance float64 `json:"average_value_max_distance"` + MaxOfMaxDistance float64 `json:"max_value_max_distance"` + MinOfMaxDistance float64 `json:"min_value_max_distance"` + DistanceLimitation float64 `json:"distance_limitation_during_preprocessing"` +} + +func newStatistic() *statistic { + return &statistic{ + Count: 0, + ValidCount: 0, + AverageNearByIDsCount: 0, + MaxNearByIDsCount: math.MinInt32, + MinNearByIDsCount: math.MaxInt32, + AverageMaxDistance: 0.0, + MaxOfMaxDistance: 0.0, + MinOfMaxDistance: math.MaxFloat64, + DistanceLimitation: 0.0, + } +} + +func (s *statistic) init() { + s.Count = 0 + s.ValidCount = 0 + s.AverageNearByIDsCount = 0 + s.MaxNearByIDsCount = math.MinInt32 + s.MinNearByIDsCount = math.MaxInt32 + s.AverageMaxDistance = 0.0 + s.MaxOfMaxDistance = 0.0 + s.MinOfMaxDistance = math.MaxFloat64 + s.DistanceLimitation = 0.0 +} + +func (s *statistic) build(m ID2NearByIDsMap, DistanceLimitation float64) *statistic { + s.init() + s.DistanceLimitation = DistanceLimitation + + totalNearByIDsCount := 0 + totalMaxDistance := 0.0 + for _, idAndDistanceArray := range m { + s.Count += 1 + if len(idAndDistanceArray) == 0 { + continue + } + s.ValidCount += 1 + + prevTotalNearByIDsCount := totalNearByIDsCount + totalNearByIDsCount += len(idAndDistanceArray) + if prevTotalNearByIDsCount > totalNearByIDsCount { + glog.Fatalf("Overflow during accumulate totalNearByIDsCount, before accumulate value = %v, after add %v new value is %v\n", + prevTotalNearByIDsCount, len(idAndDistanceArray), totalNearByIDsCount) + } + s.MaxNearByIDsCount = max(s.MaxNearByIDsCount, len(idAndDistanceArray)) + s.MinNearByIDsCount = min(s.MinNearByIDsCount, len(idAndDistanceArray)) + + prevTotalMaxDistance := totalMaxDistance + + maxDistance := 0.0 + for _, item := range idAndDistanceArray { + maxDistance = math.Max(maxDistance, item.Distance) + } + + totalMaxDistance += maxDistance + if prevTotalMaxDistance > totalMaxDistance { + glog.Fatalf("Overflow during accumulate totalMaxDistance, before accumulate value = %#v, after add %v new value is %#v\n", + prevTotalMaxDistance, maxDistance, totalMaxDistance) + } + s.MaxOfMaxDistance = math.Max(s.MaxOfMaxDistance, maxDistance) + s.MinOfMaxDistance = math.Min(s.MinOfMaxDistance, maxDistance) + } + + s.AverageNearByIDsCount = totalNearByIDsCount / s.ValidCount + s.AverageMaxDistance = totalMaxDistance / (float64)(s.ValidCount) + + glog.Infof("Build statistic for ID2NearByIDsMap finished. %+v\n", s) + return s +} + +func (s *statistic) dump(folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath += api.Slash + } + + file, err := json.Marshal(*s) + if err != nil { + glog.Errorf("Marshal object %+v failed with error %+v\n", s, err) + return err + } + + if err := ioutil.WriteFile(folderPath+statisticFileName, file, 0644); err != nil { + glog.Errorf("Dump %s to %s failed with error %+v\n", statisticFileName, folderPath, err) + return err + } + + glog.Infof("Successfully dump statistic file of %s to %s\n", statisticFileName, folderPath) + return nil +} + +func (s *statistic) load(folderPath string) error { + if !strings.HasSuffix(folderPath, api.Slash) { + folderPath += api.Slash + } + + file, err := ioutil.ReadFile(folderPath + statisticFileName) + if err != nil { + glog.Errorf("Load %s from %s failed with error %+v\n", statisticFileName, folderPath, err) + return err + } + + err = json.Unmarshal([]byte(file), s) + if err != nil { + glog.Errorf("Unmarshal statistic file %s from %s failed with error %+v\n", statisticFileName, folderPath, err) + return err + } + + return nil +} + +// max returns the larger of x or y. +func max(x, y int) int { + if x < y { + return y + } + return x +} + +// min returns the smaller of x or y. +func min(x, y int) int { + if x > y { + return y + } + return x +} diff --git a/integration/service/connectivitymap/statistic_test.go b/integration/service/connectivitymap/statistic_test.go new file mode 100644 index 00000000000..b0fdd4dfb3f --- /dev/null +++ b/integration/service/connectivitymap/statistic_test.go @@ -0,0 +1,41 @@ +package connectivitymap + +import ( + "reflect" + "testing" +) + +var fakeDistanceLimit float64 = 123 +var fakeStatisticResult1 = statistic{ + Count: 5, + ValidCount: 5, + AverageNearByIDsCount: 2, + MaxNearByIDsCount: 4, + MinNearByIDsCount: 2, + AverageMaxDistance: 15, + MaxOfMaxDistance: 23, + MinOfMaxDistance: 5, + DistanceLimitation: 123, +} + +func TestStatisticBuild(t *testing.T) { + cases := []struct { + id2NearByIDsMap ID2NearByIDsMap + distanceLimit float64 + expect *statistic + }{ + { + id2NearByIDsMap: fakeID2NearByIDsMap1, + distanceLimit: fakeDistanceLimit, + expect: &fakeStatisticResult1, + }, + } + + for _, c := range cases { + actual := newStatistic().build(c.id2NearByIDsMap, c.distanceLimit) + if !reflect.DeepEqual(actual, c.expect) { + t.Errorf("Incorrect statistic build() result, expect \n%+v \nbut got \n%+v\n", c.expect, actual) + } + } + +} diff --git a/integration/service/spatialindexer/interface.go b/integration/service/spatialindexer/interface.go index 1f4b885c9e7..9ca5c11cec6 100644 --- a/integration/service/spatialindexer/interface.go +++ b/integration/service/spatialindexer/interface.go @@ -32,7 +32,7 @@ const UnlimitedCount = math.MaxInt32 type Finder interface { // FindNearByPointIDs returns a group of points near to given center location - FindNearByPointIDs(center Location, radius float64, limitCount int) []PointInfo + FindNearByPointIDs(center Location, radius float64, limitCount int) []*PointInfo } // Ranker used to ranking a group of points diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go index 2abee9646b6..94697efe07f 100644 --- a/integration/service/spatialindexer/s2indexer/indexer.go +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -80,7 +80,7 @@ func (indexer *S2Indexer) IteratePoints() <-chan spatialindexer.PointInfo { return pointsC } -func (indexer *S2Indexer) FindNearByPointIDs(center spatialindexer.Location, radius float64, limitCount int) []spatialindexer.PointInfo { +func (indexer *S2Indexer) FindNearByPointIDs(center spatialindexer.Location, radius float64, limitCount int) []*spatialindexer.PointInfo { if !indexer.isInit() { glog.Warning("S2Indexer is empty, try to Build() with correct input file first.\n") return nil diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go index d64b997d284..9c5c1638dfd 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query.go @@ -24,8 +24,8 @@ func queryNearByS2Cells(point spatialindexer.Location, radiusInMeters float64) [ return ([]s2.CellID)(cellUnion) } -func queryNearByPoints(indexer *S2Indexer, point spatialindexer.Location, radius float64) []spatialindexer.PointInfo { - var result []spatialindexer.PointInfo +func queryNearByPoints(indexer *S2Indexer, point spatialindexer.Location, radius float64) []*spatialindexer.PointInfo { + var result []*spatialindexer.PointInfo cellIDs := queryNearByS2Cells(point, radius) @@ -42,7 +42,7 @@ func queryNearByPoints(indexer *S2Indexer, point spatialindexer.Location, radius continue } - result = append(result, spatialindexer.PointInfo{ + result = append(result, &spatialindexer.PointInfo{ ID: pointID, Location: location, }) diff --git a/integration/service/spatialindexer/s2indexer/spatial_query_test.go b/integration/service/spatialindexer/s2indexer/spatial_query_test.go index c444a8f0d8a..191fa503b9f 100644 --- a/integration/service/spatialindexer/s2indexer/spatial_query_test.go +++ b/integration/service/spatialindexer/s2indexer/spatial_query_test.go @@ -74,15 +74,15 @@ func TestSpatialIndexQuery1(t *testing.T) { Lon: -121.969861, } - expect := []spatialindexer.PointInfo{ - spatialindexer.PointInfo{ + expect := []*spatialindexer.PointInfo{ + &spatialindexer.PointInfo{ ID: 1, Location: spatialindexer.Location{ Lat: 37.402701, Lon: -121.974096, }, }, - spatialindexer.PointInfo{ + &spatialindexer.PointInfo{ ID: 2, Location: spatialindexer.Location{ Lat: 37.40353, From 7da9a68eb9a9c17ea1510eca684e90e978208c74 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Wed, 1 Apr 2020 14:13:17 -0700 Subject: [PATCH 21/27] fit: implment builder with taskspool issue: https://github.com/Telenav/osrm-backend/issues/238 --- .../service/connectivitymap/builder.go | 143 ++++++++++++++++-- .../service/connectivitymap/builder_test.go | 101 +++++++++++++ .../service/spatialindexer/iterface_mock.go | 128 ++++++++++++++++ 3 files changed, 359 insertions(+), 13 deletions(-) create mode 100644 integration/service/connectivitymap/builder_test.go create mode 100644 integration/service/spatialindexer/iterface_mock.go diff --git a/integration/service/connectivitymap/builder.go b/integration/service/connectivitymap/builder.go index b0ce5594d19..b8955a46ed8 100644 --- a/integration/service/connectivitymap/builder.go +++ b/integration/service/connectivitymap/builder.go @@ -1,22 +1,148 @@ package connectivitymap -import "github.com/Telenav/osrm-backend/integration/service/spatialindexer" +import ( + "sync" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/golang/glog" +) type connectivityMapBuilder struct { iterator spatialindexer.PointsIterator finder spatialindexer.Finder ranker spatialindexer.Ranker distanceLimit float64 + id2NearbyIDs ID2NearByIDsMap + + numOfWorker int + dispatchWaitGroup *sync.WaitGroup + workerWaitGroup *sync.WaitGroup + aggregatorWaitGroup *sync.WaitGroup + tasks4WorkerC []chan spatialindexer.PointInfo + aggregatorC chan placeIDWithNearByPlaceIDs } func newConnectivityMapBuilder(iterator spatialindexer.PointsIterator, finder spatialindexer.Finder, - ranker spatialindexer.Ranker, distanceLimit float64) *connectivityMapBuilder { - return &connectivityMapBuilder{ + ranker spatialindexer.Ranker, distanceLimit float64, numOfWorker int) *connectivityMapBuilder { + builder := &connectivityMapBuilder{ iterator: iterator, finder: finder, ranker: ranker, distanceLimit: distanceLimit, + id2NearbyIDs: make(ID2NearByIDsMap), + + numOfWorker: numOfWorker, + dispatchWaitGroup: &sync.WaitGroup{}, + workerWaitGroup: &sync.WaitGroup{}, + aggregatorWaitGroup: &sync.WaitGroup{}, + tasks4WorkerC: make([]chan spatialindexer.PointInfo, numOfWorker), + aggregatorC: make(chan placeIDWithNearByPlaceIDs, 10000), + } + + if numOfWorker < 1 { + glog.Fatal("numOfWorker should never be smaller than 1, recommend using NumCPU()\n") + } + + for i := range builder.tasks4WorkerC { + builder.tasks4WorkerC[i] = make(chan spatialindexer.PointInfo, 50) + } + + return builder +} + +/* + -> tasksChannel ---> worker + / \ + / \ +Input Iterator ==> dispatcher ---> tasksChannel ---> worker ---> aggregatorChannel -> iterate and put result to map + \ / + \ / + -> tasksChannel ---> worker +*/ + +func (builder *connectivityMapBuilder) build() ID2NearByIDsMap { + builder.dispatch() + builder.process() + builder.aggregate() + builder.wait() + + return builder.id2NearbyIDs +} + +func (builder *connectivityMapBuilder) dispatch() { + builder.dispatchWaitGroup.Add(1) + + go func() { + counter := 0 + for p := range builder.iterator.IteratePoints() { + builder.tasks4WorkerC[counter%builder.numOfWorker] <- p + counter += 1 + } + + for i := 0; i < builder.numOfWorker; i++ { + close(builder.tasks4WorkerC[i]) + } + + glog.Infof("builder::dispatch is finished. Total input is %d.\n", counter) + builder.dispatchWaitGroup.Done() + }() +} + +func (builder *connectivityMapBuilder) process() { + for i := 0; i < builder.numOfWorker; i++ { + builder.workerWaitGroup.Add(1) + go builder.work(builder.workerWaitGroup, i, builder.tasks4WorkerC[i], builder.aggregatorC) } + + glog.Infof("builder::process is finished, %d worker is started.\n", builder.numOfWorker) +} + +func (builder *connectivityMapBuilder) work(wg *sync.WaitGroup, workerID int, source <-chan spatialindexer.PointInfo, sink chan<- placeIDWithNearByPlaceIDs) { + defer wg.Done() + + counter := 0 + for p := range source { + counter += 1 + nearbyIDs := builder.finder.FindNearByPointIDs(p.Location, builder.distanceLimit, spatialindexer.UnlimitedCount) + rankedResults := builder.ranker.RankPointIDsByGreatCircleDistance(p.Location, nearbyIDs) + + ids := make([]IDAndDistance, 0, len(rankedResults)) + for _, r := range rankedResults { + ids = append(ids, IDAndDistance{ + ID: r.ID, + Distance: r.Distance, + }) + } + + sink <- placeIDWithNearByPlaceIDs{ + id: p.ID, + ids: ids, + } + } + + glog.Infof("Worker %d finished handling %d tasks.\n", workerID, counter) +} + +func (builder *connectivityMapBuilder) aggregate() { + builder.aggregatorWaitGroup.Add(1) + + go func() { + counter := 0 + for item := range builder.aggregatorC { + counter += 1 + builder.id2NearbyIDs[item.id] = item.ids + } + + glog.Infof("Aggregation is finished with %d items.\n", counter) + builder.aggregatorWaitGroup.Done() + }() +} + +func (builder *connectivityMapBuilder) wait() { + builder.dispatchWaitGroup.Wait() + builder.workerWaitGroup.Wait() + close(builder.aggregatorC) + builder.aggregatorWaitGroup.Wait() } type placeIDWithNearByPlaceIDs struct { @@ -24,8 +150,7 @@ type placeIDWithNearByPlaceIDs struct { ids []IDAndDistance } -// todo: use task pool -func (builder *connectivityMapBuilder) build() ID2NearByIDsMap { +func (builder *connectivityMapBuilder) buildViaSingleGoroutine() ID2NearByIDsMap { internalResult := make(chan placeIDWithNearByPlaceIDs, 10000) m := make(ID2NearByIDsMap) @@ -55,11 +180,3 @@ func (builder *connectivityMapBuilder) build() ID2NearByIDsMap { return m } - -// func (builder *connectivityMapBuilder) dump(folderPath string, m ID2NearByIDsMap) { - -// } - -// func (builder *connectivityMapBuilder) load(folderPath string) (ID2NearByIDsMap, float64) { - -// } diff --git a/integration/service/connectivitymap/builder_test.go b/integration/service/connectivitymap/builder_test.go new file mode 100644 index 00000000000..670afa743e2 --- /dev/null +++ b/integration/service/connectivitymap/builder_test.go @@ -0,0 +1,101 @@ +package connectivitymap + +import ( + "reflect" + "runtime" + "testing" + + "github.com/Telenav/osrm-backend/integration/service/spatialindexer" + "github.com/Telenav/osrm-backend/integration/service/spatialindexer/ranker" +) + +// Mock env: +// Iterator returns 100 of fixed location point(IDs are 1000, 1001, 1002, ...) +// Finder returns fixed array result(10 points, IDs are 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ) +// Ranker use great circle distance to calculate +// Expect result: +// build() generate same result as pre-calculated map +// map[1000] = {results of 10 points} +// map[1001] = {results of 10 points} +// ... +// map[1099] = {results of 10 points} +func TestBuilderWithMockIteratorAndFinder(t *testing.T) { + builder := newConnectivityMapBuilder(&spatialindexer.MockOneHundredPointsIterator{}, + &spatialindexer.MockFinder{}, + ranker.CreateRanker(ranker.SimpleRanker, nil), + 100, + runtime.NumCPU()) + + actual := builder.build() + + // construct expect map + expect := make(ID2NearByIDsMap) + + var idAndDistanceArray = []IDAndDistance{ + IDAndDistance{ + ID: 3, + Distance: 345.220003472554, + }, + IDAndDistance{ + ID: 2, + Distance: 402.8536530341791, + }, + IDAndDistance{ + ID: 4, + Distance: 1627.1858848458571, + }, + IDAndDistance{ + ID: 5, + Distance: 4615.586636153461, + }, + IDAndDistance{ + ID: 1, + Distance: 5257.70008125706, + }, + IDAndDistance{ + ID: 6, + Distance: 6888.7486674247, + }, + IDAndDistance{ + ID: 7, + Distance: 7041.893747628621, + }, + IDAndDistance{ + ID: 10, + Distance: 8622.213424347745, + }, + IDAndDistance{ + ID: 9, + Distance: 9438.804320070916, + }, + IDAndDistance{ + ID: 8, + Distance: 9897.44482638937, + }, + } + + for i := 0; i < 100; i++ { + index := i + 1000 + expect[(spatialindexer.PointID(index))] = idAndDistanceArray + } + + if !reflect.DeepEqual(actual, expect) { + t.Errorf("Failed to pass TestBuilder with mock data, \nexpect \n%+v\n but got \n%v\n", expect, actual) + } +} + +func TestBuilderWithSingleWorker(t *testing.T) { + builder := newConnectivityMapBuilder(&spatialindexer.MockOneHundredPointsIterator{}, + &spatialindexer.MockFinder{}, + ranker.CreateRanker(ranker.SimpleRanker, nil), + 100, + runtime.NumCPU()) + + actual := builder.build() + + expect := builder.buildViaSingleGoroutine() + + if !reflect.DeepEqual(actual, expect) { + t.Errorf("Failed to pass TestBuilder with mock data, \nexpect \n%+v\n but got \n%v\n", expect, actual) + } +} diff --git a/integration/service/spatialindexer/iterface_mock.go b/integration/service/spatialindexer/iterface_mock.go new file mode 100644 index 00000000000..617cd64afd9 --- /dev/null +++ b/integration/service/spatialindexer/iterface_mock.go @@ -0,0 +1,128 @@ +package spatialindexer + +var mockPlaceInfo1 = []*PointInfo{ + &PointInfo{ + ID: 1, + Location: Location{ + Lat: 37.355204, + Lon: -121.953901, + }, + }, + &PointInfo{ + ID: 2, + Location: Location{ + Lat: 37.399331, + Lon: -121.981193, + }, + }, + &PointInfo{ + ID: 3, + Location: Location{ + Lat: 37.401948, + Lon: -121.977384, + }, + }, + &PointInfo{ + ID: 4, + Location: Location{ + Lat: 37.407082, + Lon: -121.991937, + }, + }, + &PointInfo{ + ID: 5, + Location: Location{ + Lat: 37.407277, + Lon: -121.925482, + }, + }, + &PointInfo{ + ID: 6, + Location: Location{ + Lat: 37.375024, + Lon: -121.904706, + }, + }, + &PointInfo{ + ID: 7, + Location: Location{ + Lat: 37.359592, + Lon: -121.914164, + }, + }, + &PointInfo{ + ID: 8, + Location: Location{ + Lat: 37.366023, + Lon: -122.080777, + }, + }, + &PointInfo{ + ID: 9, + Location: Location{ + Lat: 37.368453, + Lon: -122.076400, + }, + }, + &PointInfo{ + ID: 10, + Location: Location{ + Lat: 37.373546, + Lon: -122.068904, + }, + }, +} + +// MockPointsIterator implements Finder's interface +type MockFinder struct { +} + +// FindNearByPointIDs returns mock result +func (finder *MockFinder) FindNearByPointIDs(center Location, radius float64, limitCount int) []*PointInfo { + return mockPlaceInfo1 +} + +// MockPointsIterator implements PointsIterator's interface +type MockPointsIterator struct { +} + +// IteratePoints() iterate places with mock data +func (iterator *MockPointsIterator) IteratePoints() <-chan PointInfo { + pointInfoC := make(chan PointInfo, len(mockPlaceInfo1)) + + go func() { + for _, item := range mockPlaceInfo1 { + pointInfoC <- *item + } + + close(pointInfoC) + }() + + return pointInfoC +} + +// MockOneHundredPointsIterator implements PointsIterator's interface +type MockOneHundredPointsIterator struct { +} + +// IteratePoints() iterate places with mock data +func (iterator *MockOneHundredPointsIterator) IteratePoints() <-chan PointInfo { + pointInfoC := make(chan PointInfo, 100) + + go func() { + for i := 0; i < 100; i++ { + id := (PointID)(i + 1000) + pointInfoC <- PointInfo{ + ID: id, + Location: Location{ + Lat: 37.398896, + Lon: -121.976665, + }, + } + } + + close(pointInfoC) + }() + + return pointInfoC +} From f0b5886579d8dfe68ffabf1dcc3d72ad5a50d107 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Wed, 1 Apr 2020 14:23:56 -0700 Subject: [PATCH 22/27] fix: update code based on comments --- integration/service/spatialindexer/iterface_mock.go | 4 +++- integration/service/spatialindexer/ranker/ranker_factory.go | 5 ++++- integration/service/spatialindexer/s2indexer/indexer.go | 5 +++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/integration/service/spatialindexer/iterface_mock.go b/integration/service/spatialindexer/iterface_mock.go index 617cd64afd9..654edda288c 100644 --- a/integration/service/spatialindexer/iterface_mock.go +++ b/integration/service/spatialindexer/iterface_mock.go @@ -78,6 +78,7 @@ type MockFinder struct { } // FindNearByPointIDs returns mock result +// It returns 10 places defined in mockPlaceInfo1 func (finder *MockFinder) FindNearByPointIDs(center Location, radius float64, limitCount int) []*PointInfo { return mockPlaceInfo1 } @@ -105,7 +106,8 @@ func (iterator *MockPointsIterator) IteratePoints() <-chan PointInfo { type MockOneHundredPointsIterator struct { } -// IteratePoints() iterate places with mock data +// IteratePoints() iterate places with mock data. +// It returns {ID:1000, fixed location}, {ID:1001, fixed location}, ... {ID:1099, fixed location} func (iterator *MockOneHundredPointsIterator) IteratePoints() <-chan PointInfo { pointInfoC := make(chan PointInfo, 100) diff --git a/integration/service/spatialindexer/ranker/ranker_factory.go b/integration/service/spatialindexer/ranker/ranker_factory.go index a6bd0776975..4ae51344b98 100644 --- a/integration/service/spatialindexer/ranker/ranker_factory.go +++ b/integration/service/spatialindexer/ranker/ranker_factory.go @@ -6,10 +6,13 @@ import ( ) const ( - SimpleRanker = "SimpleRanker" + // SimpleRanker implements Raner's interface based on great circle distance + SimpleRanker = "SimpleRanker" + // OSRMBasedRanker implements Raner's interface based on OSRM OSRMBasedRanker = "OSRMBasedRanker" ) +// CreateRanker creates implementations of interface Ranker func CreateRanker(rankerType string, oc *osrmconnector.OSRMConnector) spatialindexer.Ranker { switch rankerType { case SimpleRanker: diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go index 94697efe07f..24ea20ddaea 100644 --- a/integration/service/spatialindexer/s2indexer/indexer.go +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -80,8 +80,9 @@ func (indexer *S2Indexer) IteratePoints() <-chan spatialindexer.PointInfo { return pointsC } +// FindNearByPointIDs returns nearby points for given center and conditions func (indexer *S2Indexer) FindNearByPointIDs(center spatialindexer.Location, radius float64, limitCount int) []*spatialindexer.PointInfo { - if !indexer.isInit() { + if !indexer.isInitialized() { glog.Warning("S2Indexer is empty, try to Build() with correct input file first.\n") return nil } @@ -104,7 +105,7 @@ func (indexer S2Indexer) getPointIDsByS2CellID(cellid s2.CellID) ([]spatialindex return pointIDs, ok } -func (indexer S2Indexer) isInit() bool { +func (indexer S2Indexer) isInitialized() bool { return len(indexer.cellID2PointIDs) != 0 && len(indexer.pointID2Location) != 0 } From bd4ba1537a56c03fd84a4f43aeb0aa3ff13fa9bb Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Wed, 1 Apr 2020 14:52:18 -0700 Subject: [PATCH 23/27] fix: update code based on self review --- integration/service/connectivitymap/builder.go | 7 ++++--- .../service/connectivitymap/connectivity_map.go | 17 ++++++++++++----- .../service/connectivitymap/dumper_test.go | 4 ++-- .../service/spatialindexer/s2indexer/indexer.go | 5 ----- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/integration/service/connectivitymap/builder.go b/integration/service/connectivitymap/builder.go index b8955a46ed8..36e4964febe 100644 --- a/integration/service/connectivitymap/builder.go +++ b/integration/service/connectivitymap/builder.go @@ -94,7 +94,7 @@ func (builder *connectivityMapBuilder) process() { go builder.work(builder.workerWaitGroup, i, builder.tasks4WorkerC[i], builder.aggregatorC) } - glog.Infof("builder::process is finished, %d worker is started.\n", builder.numOfWorker) + glog.Infof("builder::process is finished, start number of %d workers.\n", builder.numOfWorker) } func (builder *connectivityMapBuilder) work(wg *sync.WaitGroup, workerID int, source <-chan spatialindexer.PointInfo, sink chan<- placeIDWithNearByPlaceIDs) { @@ -120,7 +120,7 @@ func (builder *connectivityMapBuilder) work(wg *sync.WaitGroup, workerID int, so } } - glog.Infof("Worker %d finished handling %d tasks.\n", workerID, counter) + glog.Infof("Worker_%d finished handling %d tasks.\n", workerID, counter) } func (builder *connectivityMapBuilder) aggregate() { @@ -133,7 +133,7 @@ func (builder *connectivityMapBuilder) aggregate() { builder.id2NearbyIDs[item.id] = item.ids } - glog.Infof("Aggregation is finished with %d items.\n", counter) + glog.Infof("Aggregation is finished with handling %d items.\n", counter) builder.aggregatorWaitGroup.Done() }() } @@ -151,6 +151,7 @@ type placeIDWithNearByPlaceIDs struct { } func (builder *connectivityMapBuilder) buildViaSingleGoroutine() ID2NearByIDsMap { + glog.Warning("This function is only used for compare result of worker::build().\n") internalResult := make(chan placeIDWithNearByPlaceIDs, 10000) m := make(ID2NearByIDsMap) diff --git a/integration/service/connectivitymap/connectivity_map.go b/integration/service/connectivitymap/connectivity_map.go index 9d68294ce9c..86bc671df51 100644 --- a/integration/service/connectivitymap/connectivity_map.go +++ b/integration/service/connectivitymap/connectivity_map.go @@ -14,33 +14,40 @@ type IDAndDistance struct { // ID2NearByIDsMap is a mapping between ID and its nearby IDs type ID2NearByIDsMap map[spatialindexer.PointID][]IDAndDistance +// Connectivity Map used to query connectivity for given placeID type ConnectivityMap struct { id2nearByIDs ID2NearByIDsMap distanceLimitation float64 statistic *statistic } -func (cm *ConnectivityMap) DistanceLimitation() float64 { - return cm.distanceLimitation -} - -func NewPlaceConnectivityMap(distanceLimitation float64) *ConnectivityMap { +// NewConnectivityMap creates ConnectivityMap +func NewConnectivityMap(distanceLimitation float64) *ConnectivityMap { return &ConnectivityMap{ distanceLimitation: distanceLimitation, statistic: newStatistic(), } } +// Build creates ConnectivityMap func (cm *ConnectivityMap) Build() { glog.Info("Successfully finished GenerateConnectivityMap\n") } +// Dump dump ConnectivityMap's content to given folderPath func (cm *ConnectivityMap) Dump(folderPath string) { } +// Load rebuild ConnectivityMap from dumpped data in given folderPath func (cm *ConnectivityMap) Load(folderPath string) { } +// QueryConnectivity answers connectivity query for given placeInfo func (cm *ConnectivityMap) QueryConnectivity(placeInfo spatialindexer.PointInfo, limitDistance float64) { // for each everything recorded in data, apply limit option on that } + +// DistanceLimitation tells the value used to pre-process place data +func (cm *ConnectivityMap) DistanceLimitation() float64 { + return cm.distanceLimitation +} diff --git a/integration/service/connectivitymap/dumper_test.go b/integration/service/connectivitymap/dumper_test.go index ec94f16fa6c..fc5bd68fe4a 100644 --- a/integration/service/connectivitymap/dumper_test.go +++ b/integration/service/connectivitymap/dumper_test.go @@ -8,7 +8,7 @@ import ( "github.com/golang/glog" ) -func TestDumper(t *testing.T) { +func TestDumpGivenObjectThenLoadAndThenCompareWithOriginalObject(t *testing.T) { cases := []ConnectivityMap{ ConnectivityMap{ id2nearByIDs: fakeID2NearByIDsMap1, @@ -36,7 +36,7 @@ func TestDumper(t *testing.T) { t.Errorf("During running serializeConnectivityMap for case %v, met error %v", c, err) } - actual := NewPlaceConnectivityMap(0.0) + actual := NewConnectivityMap(0.0) if err := deSerializeConnectivityMap(actual, path); err != nil { t.Errorf("During running deSerializeConnectivityMap for case %v, met error %v", c, err) } diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go index bcddac2eeaf..24ea20ddaea 100644 --- a/integration/service/spatialindexer/s2indexer/indexer.go +++ b/integration/service/spatialindexer/s2indexer/indexer.go @@ -80,14 +80,9 @@ func (indexer *S2Indexer) IteratePoints() <-chan spatialindexer.PointInfo { return pointsC } -<<<<<<< HEAD -func (indexer *S2Indexer) FindNearByPointIDs(center spatialindexer.Location, radius float64, limitCount int) []spatialindexer.PointInfo { - if !indexer.isInit() { -======= // FindNearByPointIDs returns nearby points for given center and conditions func (indexer *S2Indexer) FindNearByPointIDs(center spatialindexer.Location, radius float64, limitCount int) []*spatialindexer.PointInfo { if !indexer.isInitialized() { ->>>>>>> feature/placeconnectivitymap-impl glog.Warning("S2Indexer is empty, try to Build() with correct input file first.\n") return nil } From abad5aad396389cacd697289c40ee7bbac83f455 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Wed, 1 Apr 2020 15:15:10 -0700 Subject: [PATCH 24/27] fix: update comments. --- integration/service/connectivitymap/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/service/connectivitymap/builder.go b/integration/service/connectivitymap/builder.go index 36e4964febe..707920b5f9f 100644 --- a/integration/service/connectivitymap/builder.go +++ b/integration/service/connectivitymap/builder.go @@ -151,7 +151,7 @@ type placeIDWithNearByPlaceIDs struct { } func (builder *connectivityMapBuilder) buildViaSingleGoroutine() ID2NearByIDsMap { - glog.Warning("This function is only used for compare result of worker::build().\n") + glog.Warning("This function is only used for compare result of worker::build() in unittest.\n") internalResult := make(chan placeIDWithNearByPlaceIDs, 10000) m := make(ID2NearByIDsMap) From 4c65cb062da9eb5f7ad7545ddcf07132d0c7a482 Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Thu, 2 Apr 2020 10:39:53 -0700 Subject: [PATCH 25/27] fix: adjust code based on comments. --- .../service/connectivitymap/builder.go | 8 ++++---- .../service/connectivitymap/builder_test.go | 2 +- .../connectivitymap/connectivity_map.go | 19 ++++++++++--------- integration/service/connectivitymap/doc.go | 2 +- integration/service/connectivitymap/dumper.go | 2 +- .../service/connectivitymap/dumper_test.go | 10 +++++----- .../service/connectivitymap/statistic.go | 10 +++++----- .../service/connectivitymap/statistic_test.go | 2 +- .../{iterface_mock.go => interface_mock.go} | 0 9 files changed, 28 insertions(+), 27 deletions(-) rename integration/service/spatialindexer/{iterface_mock.go => interface_mock.go} (100%) diff --git a/integration/service/connectivitymap/builder.go b/integration/service/connectivitymap/builder.go index 707920b5f9f..b1a43806b2d 100644 --- a/integration/service/connectivitymap/builder.go +++ b/integration/service/connectivitymap/builder.go @@ -83,7 +83,7 @@ func (builder *connectivityMapBuilder) dispatch() { close(builder.tasks4WorkerC[i]) } - glog.Infof("builder::dispatch is finished. Total input is %d.\n", counter) + glog.Infof("builder's dispatch is finished. Total input is %d.\n", counter) builder.dispatchWaitGroup.Done() }() } @@ -94,7 +94,7 @@ func (builder *connectivityMapBuilder) process() { go builder.work(builder.workerWaitGroup, i, builder.tasks4WorkerC[i], builder.aggregatorC) } - glog.Infof("builder::process is finished, start number of %d workers.\n", builder.numOfWorker) + glog.Infof("builder's process is finished, start number of %d workers.\n", builder.numOfWorker) } func (builder *connectivityMapBuilder) work(wg *sync.WaitGroup, workerID int, source <-chan spatialindexer.PointInfo, sink chan<- placeIDWithNearByPlaceIDs) { @@ -150,8 +150,8 @@ type placeIDWithNearByPlaceIDs struct { ids []IDAndDistance } -func (builder *connectivityMapBuilder) buildViaSingleGoroutine() ID2NearByIDsMap { - glog.Warning("This function is only used for compare result of worker::build() in unittest.\n") +func (builder *connectivityMapBuilder) buildInSerial() ID2NearByIDsMap { + glog.Warning("This function is only used for compare result of worker's build().\n`") internalResult := make(chan placeIDWithNearByPlaceIDs, 10000) m := make(ID2NearByIDsMap) diff --git a/integration/service/connectivitymap/builder_test.go b/integration/service/connectivitymap/builder_test.go index 670afa743e2..35c55b8619c 100644 --- a/integration/service/connectivitymap/builder_test.go +++ b/integration/service/connectivitymap/builder_test.go @@ -93,7 +93,7 @@ func TestBuilderWithSingleWorker(t *testing.T) { actual := builder.build() - expect := builder.buildViaSingleGoroutine() + expect := builder.buildInSerial() if !reflect.DeepEqual(actual, expect) { t.Errorf("Failed to pass TestBuilder with mock data, \nexpect \n%+v\n but got \n%v\n", expect, actual) diff --git a/integration/service/connectivitymap/connectivity_map.go b/integration/service/connectivitymap/connectivity_map.go index 86bc671df51..02497a77c61 100644 --- a/integration/service/connectivitymap/connectivity_map.go +++ b/integration/service/connectivitymap/connectivity_map.go @@ -16,16 +16,16 @@ type ID2NearByIDsMap map[spatialindexer.PointID][]IDAndDistance // Connectivity Map used to query connectivity for given placeID type ConnectivityMap struct { - id2nearByIDs ID2NearByIDsMap - distanceLimitation float64 - statistic *statistic + id2nearByIDs ID2NearByIDsMap + maxRange float64 + statistic *statistic } // NewConnectivityMap creates ConnectivityMap -func NewConnectivityMap(distanceLimitation float64) *ConnectivityMap { +func NewConnectivityMap(maxRange float64) *ConnectivityMap { return &ConnectivityMap{ - distanceLimitation: distanceLimitation, - statistic: newStatistic(), + maxRange: maxRange, + statistic: newStatistic(), } } @@ -47,7 +47,8 @@ func (cm *ConnectivityMap) QueryConnectivity(placeInfo spatialindexer.PointInfo, // for each everything recorded in data, apply limit option on that } -// DistanceLimitation tells the value used to pre-process place data -func (cm *ConnectivityMap) DistanceLimitation() float64 { - return cm.distanceLimitation +// MaxRange tells the value used to pre-process place data. +// MaxRange means the maximum distance in meters could be reached from current location. +func (cm *ConnectivityMap) MaxRange() float64 { + return cm.maxRange } diff --git a/integration/service/connectivitymap/doc.go b/integration/service/connectivitymap/doc.go index a6237bd377e..ffb4f3a6bb1 100644 --- a/integration/service/connectivitymap/doc.go +++ b/integration/service/connectivitymap/doc.go @@ -1,5 +1,5 @@ /* -package placeconnectivitymap provides connectivity information for place graphs. +Package connectivitymap provides connectivity information for place graphs. For example, given following graph diff --git a/integration/service/connectivitymap/dumper.go b/integration/service/connectivitymap/dumper.go index b8d21441f51..d377c21cc6a 100644 --- a/integration/service/connectivitymap/dumper.go +++ b/integration/service/connectivitymap/dumper.go @@ -43,7 +43,7 @@ func deSerializeConnectivityMap(cm *ConnectivityMap, folderPath string) error { if err := cm.statistic.load(folderPath); err != nil { return err } - cm.distanceLimitation = cm.statistic.DistanceLimitation + cm.maxRange = cm.statistic.MaxRange return nil } diff --git a/integration/service/connectivitymap/dumper_test.go b/integration/service/connectivitymap/dumper_test.go index fc5bd68fe4a..bce177a0400 100644 --- a/integration/service/connectivitymap/dumper_test.go +++ b/integration/service/connectivitymap/dumper_test.go @@ -11,9 +11,9 @@ import ( func TestDumpGivenObjectThenLoadAndThenCompareWithOriginalObject(t *testing.T) { cases := []ConnectivityMap{ ConnectivityMap{ - id2nearByIDs: fakeID2NearByIDsMap1, - distanceLimitation: fakeDistanceLimit, - statistic: &fakeStatisticResult1, + id2nearByIDs: fakeID2NearByIDsMap1, + maxRange: fakeDistanceLimit, + statistic: &fakeStatisticResult1, }, } @@ -45,8 +45,8 @@ func TestDumpGivenObjectThenLoadAndThenCompareWithOriginalObject(t *testing.T) { t.Errorf("Expect result \n%+v but got \n%+v\n", c.id2nearByIDs, actual.id2nearByIDs) } - if !reflect.DeepEqual(actual.distanceLimitation, c.distanceLimitation) { - t.Errorf("Expect result \n%+v but got \n%+v\n", c.distanceLimitation, actual.distanceLimitation) + if !reflect.DeepEqual(actual.maxRange, c.maxRange) { + t.Errorf("Expect result \n%+v but got \n%+v\n", c.maxRange, actual.maxRange) } if !reflect.DeepEqual(actual.statistic, c.statistic) { diff --git a/integration/service/connectivitymap/statistic.go b/integration/service/connectivitymap/statistic.go index 5df25774f4d..aa2a850a6c2 100644 --- a/integration/service/connectivitymap/statistic.go +++ b/integration/service/connectivitymap/statistic.go @@ -21,7 +21,7 @@ type statistic struct { AverageMaxDistance float64 `json:"average_value_max_distance"` MaxOfMaxDistance float64 `json:"max_value_max_distance"` MinOfMaxDistance float64 `json:"min_value_max_distance"` - DistanceLimitation float64 `json:"distance_limitation_during_preprocessing"` + MaxRange float64 `json:"maxrange_set_by_preprocessing"` } func newStatistic() *statistic { @@ -34,7 +34,7 @@ func newStatistic() *statistic { AverageMaxDistance: 0.0, MaxOfMaxDistance: 0.0, MinOfMaxDistance: math.MaxFloat64, - DistanceLimitation: 0.0, + MaxRange: 0.0, } } @@ -47,12 +47,12 @@ func (s *statistic) init() { s.AverageMaxDistance = 0.0 s.MaxOfMaxDistance = 0.0 s.MinOfMaxDistance = math.MaxFloat64 - s.DistanceLimitation = 0.0 + s.MaxRange = 0.0 } -func (s *statistic) build(m ID2NearByIDsMap, DistanceLimitation float64) *statistic { +func (s *statistic) build(m ID2NearByIDsMap, MaxRange float64) *statistic { s.init() - s.DistanceLimitation = DistanceLimitation + s.MaxRange = MaxRange totalNearByIDsCount := 0 totalMaxDistance := 0.0 diff --git a/integration/service/connectivitymap/statistic_test.go b/integration/service/connectivitymap/statistic_test.go index b0fdd4dfb3f..957d1d966f2 100644 --- a/integration/service/connectivitymap/statistic_test.go +++ b/integration/service/connectivitymap/statistic_test.go @@ -15,7 +15,7 @@ var fakeStatisticResult1 = statistic{ AverageMaxDistance: 15, MaxOfMaxDistance: 23, MinOfMaxDistance: 5, - DistanceLimitation: 123, + MaxRange: 123, } func TestStatisticBuild(t *testing.T) { diff --git a/integration/service/spatialindexer/iterface_mock.go b/integration/service/spatialindexer/interface_mock.go similarity index 100% rename from integration/service/spatialindexer/iterface_mock.go rename to integration/service/spatialindexer/interface_mock.go From 3a9474fe58433af61cb506c887fbd8a7758c4f8a Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Fri, 3 Apr 2020 07:23:55 -0700 Subject: [PATCH 26/27] fix: rafactor builder, remove unnecessary dispatcher. issue: https://github.com/Telenav/osrm-backend/issues/238 --- .../service/connectivitymap/builder.go | 55 +++++-------------- 1 file changed, 14 insertions(+), 41 deletions(-) diff --git a/integration/service/connectivitymap/builder.go b/integration/service/connectivitymap/builder.go index b1a43806b2d..39c13f858fe 100644 --- a/integration/service/connectivitymap/builder.go +++ b/integration/service/connectivitymap/builder.go @@ -15,10 +15,8 @@ type connectivityMapBuilder struct { id2NearbyIDs ID2NearByIDsMap numOfWorker int - dispatchWaitGroup *sync.WaitGroup workerWaitGroup *sync.WaitGroup aggregatorWaitGroup *sync.WaitGroup - tasks4WorkerC []chan spatialindexer.PointInfo aggregatorC chan placeIDWithNearByPlaceIDs } @@ -32,10 +30,8 @@ func newConnectivityMapBuilder(iterator spatialindexer.PointsIterator, finder sp id2NearbyIDs: make(ID2NearByIDsMap), numOfWorker: numOfWorker, - dispatchWaitGroup: &sync.WaitGroup{}, workerWaitGroup: &sync.WaitGroup{}, aggregatorWaitGroup: &sync.WaitGroup{}, - tasks4WorkerC: make([]chan spatialindexer.PointInfo, numOfWorker), aggregatorC: make(chan placeIDWithNearByPlaceIDs, 10000), } @@ -43,25 +39,20 @@ func newConnectivityMapBuilder(iterator spatialindexer.PointsIterator, finder sp glog.Fatal("numOfWorker should never be smaller than 1, recommend using NumCPU()\n") } - for i := range builder.tasks4WorkerC { - builder.tasks4WorkerC[i] = make(chan spatialindexer.PointInfo, 50) - } - return builder } /* - -> tasksChannel ---> worker - / \ - / \ -Input Iterator ==> dispatcher ---> tasksChannel ---> worker ---> aggregatorChannel -> iterate and put result to map - \ / - \ / - -> tasksChannel ---> worker + -> worker (fetch task -> find -> rank) + / \ + / \ + Input Iterator(channel) ---> worker (fetch task -> find -> rank) ---> aggregatorChannel -> feed to map + \ / + \ / + -> worker (fetch task -> find -> rank) */ func (builder *connectivityMapBuilder) build() ID2NearByIDsMap { - builder.dispatch() builder.process() builder.aggregate() builder.wait() @@ -69,42 +60,25 @@ func (builder *connectivityMapBuilder) build() ID2NearByIDsMap { return builder.id2NearbyIDs } -func (builder *connectivityMapBuilder) dispatch() { - builder.dispatchWaitGroup.Add(1) - - go func() { - counter := 0 - for p := range builder.iterator.IteratePoints() { - builder.tasks4WorkerC[counter%builder.numOfWorker] <- p - counter += 1 - } - - for i := 0; i < builder.numOfWorker; i++ { - close(builder.tasks4WorkerC[i]) - } - - glog.Infof("builder's dispatch is finished. Total input is %d.\n", counter) - builder.dispatchWaitGroup.Done() - }() -} - func (builder *connectivityMapBuilder) process() { + inputC := builder.iterator.IteratePoints() + for i := 0; i < builder.numOfWorker; i++ { builder.workerWaitGroup.Add(1) - go builder.work(builder.workerWaitGroup, i, builder.tasks4WorkerC[i], builder.aggregatorC) + go builder.work(i, inputC, builder.aggregatorC) } - glog.Infof("builder's process is finished, start number of %d workers.\n", builder.numOfWorker) + glog.Infof("builder start number of %d workers.\n", builder.numOfWorker) } -func (builder *connectivityMapBuilder) work(wg *sync.WaitGroup, workerID int, source <-chan spatialindexer.PointInfo, sink chan<- placeIDWithNearByPlaceIDs) { - defer wg.Done() +func (builder *connectivityMapBuilder) work(workerID int, source <-chan spatialindexer.PointInfo, sink chan<- placeIDWithNearByPlaceIDs) { + defer builder.workerWaitGroup.Done() counter := 0 for p := range source { counter += 1 nearbyIDs := builder.finder.FindNearByPointIDs(p.Location, builder.distanceLimit, spatialindexer.UnlimitedCount) - rankedResults := builder.ranker.RankPointIDsByGreatCircleDistance(p.Location, nearbyIDs) + rankedResults := builder.ranker.RankPointIDsByShortestDistance(p.Location, nearbyIDs) ids := make([]IDAndDistance, 0, len(rankedResults)) for _, r := range rankedResults { @@ -139,7 +113,6 @@ func (builder *connectivityMapBuilder) aggregate() { } func (builder *connectivityMapBuilder) wait() { - builder.dispatchWaitGroup.Wait() builder.workerWaitGroup.Wait() close(builder.aggregatorC) builder.aggregatorWaitGroup.Wait() From f55ea3669b3c4aa7966827758787fe51ef74b2cb Mon Sep 17 00:00:00 2001 From: CoderBear801 Date: Fri, 3 Apr 2020 07:32:17 -0700 Subject: [PATCH 27/27] fix: revert back ranker's change --- integration/service/connectivitymap/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/service/connectivitymap/builder.go b/integration/service/connectivitymap/builder.go index 39c13f858fe..90cb6c534d4 100644 --- a/integration/service/connectivitymap/builder.go +++ b/integration/service/connectivitymap/builder.go @@ -78,7 +78,7 @@ func (builder *connectivityMapBuilder) work(workerID int, source <-chan spatiali for p := range source { counter += 1 nearbyIDs := builder.finder.FindNearByPointIDs(p.Location, builder.distanceLimit, spatialindexer.UnlimitedCount) - rankedResults := builder.ranker.RankPointIDsByShortestDistance(p.Location, nearbyIDs) + rankedResults := builder.ranker.RankPointIDsByGreatCircleDistance(p.Location, nearbyIDs) ids := make([]IDAndDistance, 0, len(rankedResults)) for _, r := range rankedResults {