fix: make place-connectivity-gen work and add related logs.

issue: #238
Telenav · Apr 2, 2020 · b721258 · b721258
1 parent 0773592
commit b721258
Show file tree

Hide file tree

Showing 10 changed files with 128 additions and 45 deletions.
diff --git a/integration/cmd/place-connectivity-gen/flags.go b/integration/cmd/place-connectivity-gen/flags.go
@@ -1,13 +1,25 @@
 package main
 
-import "flag"
+import (
+	"flag"
+	"runtime"
+)
 
 var flags struct {
-	inputPath    string
-	outputFolder string
+	inputFile       string
+	outputFolder    string
+	numberOfWorkers int
+	maxRange        float64
 }
 
+// defaultRange indicates default max drive range in meters.
+// This value always bigger than max driven ranges for existing vehicles.
+// During query time could just take subset of pre-processed result based on real range.
+const defaultMaxRange = 800000
+
 func init() {
-	flag.StringVar(&flags.inputPath, "i", "", "path for input file in json format")
+	flag.StringVar(&flags.inputFile, "i", "", "path for input file in json format")
 	flag.StringVar(&flags.outputFolder, "o", "", "path for output folder")
+	flag.IntVar(&flags.numberOfWorkers, "num_of_workers", runtime.NumCPU(), "number of workers to build connectivity map")
+	flag.Float64Var(&flags.maxRange, "range", defaultMaxRange, "maximum drive range in meters used for preprocessing.")
 }
diff --git a/integration/cmd/place-connectivity-gen/main.go b/integration/cmd/place-connectivity-gen/main.go
@@ -1,10 +1,34 @@
 package main
 
 import (
-	"fmt"
+	"flag"
+	"os"
+	"time"
+
+	"github.com/Telenav/osrm-backend/integration/service/connectivitymap"
+	"github.com/Telenav/osrm-backend/integration/service/spatialindexer/ranker"
+	"github.com/Telenav/osrm-backend/integration/service/spatialindexer/s2indexer"
+	"github.com/golang/glog"
 )
 
 func main() {
-	// @todo: add logic to generate connectivity for charge stations
-	fmt.Print("Hello World!")
+	flag.Parse()
+	defer glog.Flush()
+	startTime := time.Now()
+
+	if flags.inputFile == "" || flags.outputFolder == "" {
+		glog.Fatal("Empty string for inputFile or outputFolder, please check your input.\n")
+	}
+
+	indexer := s2indexer.NewS2Indexer().Build(flags.inputFile)
+	if indexer == nil {
+		glog.Fatalf("Failed to build indexer, stop %s\n", os.Args[0])
+	}
+	indexer.Dump(flags.outputFolder)
+
+	connectivitymap.NewConnectivityMap(flags.maxRange).
+		Build(indexer, indexer, ranker.CreateRanker(ranker.SimpleRanker, nil), 1).
+		Dump(flags.outputFolder)
+
+	glog.Infof("%s totally takes %f seconds for processing.", os.Args[0], time.Since(startTime).Seconds())
 }
diff --git a/integration/service/connectivitymap/connectivity_map.go b/integration/service/connectivitymap/connectivity_map.go
@@ -32,15 +32,19 @@ func NewConnectivityMap(maxRange float64) *ConnectivityMap {
 // Build creates ConnectivityMap
 func (cm *ConnectivityMap) Build(iterator spatialindexer.PointsIterator, finder spatialindexer.Finder,
 	ranker spatialindexer.Ranker, numOfWorkers int) *ConnectivityMap {
+	glog.Info("Start ConnectivityMap's Build().\n")
+
 	cm.id2nearByIDs = newConnectivityMapBuilder(iterator, finder, ranker, cm.maxRange, numOfWorkers).build()
 	cm.statistic = cm.statistic.build(cm.id2nearByIDs, cm.maxRange)
 
-	glog.Info("Successfully finished Build ConnectivityMap\n")
+	glog.Info("Finished ConnectivityMap's Build().\n")
 	return cm
 }
 
 // Dump dump ConnectivityMap's content to given folderPath
 func (cm *ConnectivityMap) Dump(folderPath string) {
+	glog.Info("Start ConnectivityMap's Dump().\n")
+
 	if err := removeAllDumpFiles(folderPath); err != nil {
 		glog.Fatalf("removeAllDumpFiles for ConnectivityMap failed with error %+v\n", err)
 	}
@@ -49,16 +53,18 @@ func (cm *ConnectivityMap) Dump(folderPath string) {
 		glog.Fatalf("serializeConnectivityMap failed with error %+v\n", err)
 	}
 
-	glog.Infof("Successfully finished Dump ConnectivityMap to %s\n", folderPath)
+	glog.Infof("Finished ConnectivityMap's Dump() into %s.\n", folderPath)
 }
 
 // Load rebuild ConnectivityMap from dumpped data in given folderPath
 func (cm *ConnectivityMap) Load(folderPath string) *ConnectivityMap {
+	glog.Info("Start ConnectivityMap's Load().\n")
+
 	if err := deSerializeConnectivityMap(cm, folderPath); err != nil {
 		glog.Fatalf("deSerializeConnectivityMap failed with error %+v\n", err)
 	}
 
-	glog.Infof("Successfully finished Load ConnectivityMap from %s\n", folderPath)
+	glog.Infof("Finished ConnectivityMap's Load() from %s.\n", folderPath)
 	return cm
 }
 

diff --git a/integration/service/connectivitymap/dumper.go b/integration/service/connectivitymap/dumper.go
@@ -14,6 +14,8 @@ import (
 const id2NearByIDsMapFileName = "id2nearbyidsmap.gob"
 
 func serializeConnectivityMap(cm *ConnectivityMap, folderPath string) error {
+	glog.Info("Start serializeConnectivityMap()\n")
+
 	if !strings.HasSuffix(folderPath, api.Slash) {
 		folderPath += api.Slash
 	}
@@ -26,12 +28,14 @@ func serializeConnectivityMap(cm *ConnectivityMap, folderPath string) error {
 		return err
 	}
 
-	glog.Infof("Successfully deserialize ConnectivityMap from folder %s.\n", folderPath)
+	glog.Infof("Finished serializeConnectivityMap() to folder %s.\n", folderPath)
 
 	return nil
 }
 
 func deSerializeConnectivityMap(cm *ConnectivityMap, folderPath string) error {
+	glog.Info("Start deSerializeConnectivityMap()")
+
 	if !strings.HasSuffix(folderPath, api.Slash) {
 		folderPath += api.Slash
 	}
@@ -45,6 +49,8 @@ func deSerializeConnectivityMap(cm *ConnectivityMap, folderPath string) error {
 	}
 	cm.maxRange = cm.statistic.MaxRange
 
+	glog.Infof("Finished deSerializeConnectivityMap() to folder %s.\n", folderPath)
+
 	return nil
 }
 

diff --git a/integration/service/connectivitymap/statistic.go b/integration/service/connectivitymap/statistic.go
@@ -111,7 +111,7 @@ func (s *statistic) dump(folderPath string) error {
 		return err
 	}
 
-	glog.Infof("Successfully dump statistic file of %s to %s\n", statisticFileName, folderPath)
+	glog.Infof("Finished dumpping statistic file of %s to %s\n", statisticFileName, folderPath)
 	return nil
 }
 

diff --git a/integration/service/spatialindexer/poiloader/loader.go b/integration/service/spatialindexer/poiloader/loader.go
@@ -22,5 +22,7 @@ func LoadData(filePath string) ([]Element, error) {
 		glog.Errorf("While unmarshal json file %s, met error %v\n", filePath, err)
 		return elements, err
 	}
+
+	glog.Infof("Finished loading %d items from json file %s", len(elements), filePath)
 	return elements, nil
 }
diff --git a/integration/service/spatialindexer/poiloader/poi_format.go b/integration/service/spatialindexer/poiloader/poi_format.go
@@ -1,36 +1,38 @@
 package poiloader
 
+import "encoding/json"
+
 // Element represent information loaded for point record
 type Element struct {
-	ID                int64             `json:"id"`
-	VendorCode        VendorCode        `json:"vendor_code"`
-	VendorPoiID       string            `json:"vendor_poi_id"`
-	Lat               float64           `json:"lat"`
-	Lon               float64           `json:"lon"`
-	NavLat            float64           `json:"nav_lat"`
-	NavLon            float64           `json:"nav_lon"`
-	DisLat            float64           `json:"dis_lat"`
-	DisLon            float64           `json:"dis_lon"`
-	MapLinkID         int64             `json:"map_link_id"`
-	SideOfStreet      string            `json:"side_of_street"`
-	Country           Country           `json:"country"`
-	SpaceID           SpaceID           `json:"space_id"`
-	AirportCode       string            `json:"airport_code"`
-	IsNational        bool              `json:"is_national"`
-	IsStateImportance bool              `json:"is_state_importance"`
-	IsCityImportance  bool              `json:"is_city_importance"`
-	Fax               string            `json:"fax"`
-	Phone             *CategoryIDGather `json:"phone"`
-	EncodedPhone      string            `json:"encoded_phone"`
-	Email             string            `json:"email"`
-	WebURL            string            `json:"web_url"`
-	CategoryIDGather  *CategoryIDGather `json:"category_id_gather"`
-	ChainGather       string            `json:"chain_gather"`
-	RawCategoryGather string            `json:"raw_category_gather"`
-	ChildGather       string            `json:"child_gather"`
-	ParentGather      string            `json:"parent_gather"`
-	Hilbert           float64           `json:"hilbert"`
-	Amenity           Amenity           `json:"amenity"`
+	ID                int64       `json:"id"`
+	VendorCode        VendorCode  `json:"vendor_code"`
+	VendorPoiID       string      `json:"vendor_poi_id"`
+	Lat               float64     `json:"lat"`
+	Lon               float64     `json:"lon"`
+	NavLat            float64     `json:"nav_lat"`
+	NavLon            float64     `json:"nav_lon"`
+	DisLat            float64     `json:"dis_lat"`
+	DisLon            float64     `json:"dis_lon"`
+	MapLinkID         int64       `json:"map_link_id"`
+	SideOfStreet      string      `json:"side_of_street"`
+	Country           Country     `json:"country"`
+	SpaceID           SpaceID     `json:"space_id"`
+	AirportCode       string      `json:"airport_code"`
+	IsNational        bool        `json:"is_national"`
+	IsStateImportance bool        `json:"is_state_importance"`
+	IsCityImportance  bool        `json:"is_city_importance"`
+	Fax               string      `json:"fax"`
+	Phone             json.Number `json:"phone"`
+	EncodedPhone      string      `json:"encoded_phone"`
+	Email             string      `json:"email"`
+	WebURL            string      `json:"web_url"`
+	CategoryIDGather  json.Number `json:"category_id_gather"`
+	ChainGather       string      `json:"chain_gather"`
+	RawCategoryGather string      `json:"raw_category_gather"`
+	ChildGather       string      `json:"child_gather"`
+	ParentGather      string      `json:"parent_gather"`
+	Hilbert           float64     `json:"hilbert"`
+	Amenity           Amenity     `json:"amenity"`
 }
 
 // Amenity is defined by OSM format: https://wiki.openstreetmap.org/wiki/Tag:amenity%3Dcharging_station

diff --git a/integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go b/integration/service/spatialindexer/ranker/rank_by_great_circle_distance_impl.go
@@ -8,7 +8,7 @@ import (
 
 func rankPointsByGreatCircleDistanceToCenter(center spatialindexer.Location, targets []*spatialindexer.PointInfo) []*spatialindexer.RankedPointInfo {
 	if len(targets) == 0 {
-		glog.Warning("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty\n")
+		glog.Warningf("When try to rankPointsByGreatCircleDistanceToCenter, input array is empty, center = %+v\n", center)
 		return nil
 	}
 

diff --git a/integration/service/spatialindexer/s2indexer/indexer.go b/integration/service/spatialindexer/s2indexer/indexer.go
@@ -1,27 +1,39 @@
 package s2indexer
 
 import (
+	"time"
+
 	"github.com/Telenav/osrm-backend/integration/service/spatialindexer"
 	"github.com/Telenav/osrm-backend/integration/service/spatialindexer/poiloader"
 	"github.com/golang/geo/s2"
 	"github.com/golang/glog"
 )
 
+type cellID2PointIDMap map[s2.CellID][]spatialindexer.PointID
+type pointID2LocationMap map[spatialindexer.PointID]spatialindexer.Location
+
 // S2Indexer provide spatial index ability based on google s2
 type S2Indexer struct {
-	cellID2PointIDs  map[s2.CellID][]spatialindexer.PointID
-	pointID2Location map[spatialindexer.PointID]spatialindexer.Location
+	cellID2PointIDs  cellID2PointIDMap
+	pointID2Location pointID2LocationMap
 }
 
 // NewS2Indexer generates spatial indexer based on google s2
 func NewS2Indexer() *S2Indexer {
-	return &S2Indexer{}
+	return &S2Indexer{
+		cellID2PointIDs:  make(cellID2PointIDMap),
+		pointID2Location: make(pointID2LocationMap),
+	}
 }
 
 // Build constructs S2 indexer
 func (indexer *S2Indexer) Build(filePath string) *S2Indexer {
+	glog.Info("Start Build S2Indexer.\n")
+	startTime := time.Now()
+
 	records, err := poiloader.LoadData(filePath)
 	if err != nil || len(records) == 0 {
+		glog.Error("Failed to Build S2Indexer.\n")
 		return nil
 	}
 
@@ -44,23 +56,34 @@ func (indexer *S2Indexer) Build(filePath string) *S2Indexer {
 	}
 
 	indexer.cellID2PointIDs = build(pointInfos, minS2Level, maxS2Level)
+
+	glog.Info("Finished Build() S2Indexer.\n")
+	glog.Infof("Build S2Indexer takes %f seconds.\n", time.Since(startTime).Seconds())
 	return indexer
 }
 
 // Load S2Indexer's data from contents recorded in folder
 func (indexer *S2Indexer) Load(folderPath string) *S2Indexer {
+	glog.Info("Start S2Indexer's Load().\n")
+
 	if err := deSerializeS2Indexer(indexer, folderPath); err != nil {
 		glog.Errorf("Load S2Indexer's data from folder %s failed, err=%v\n", folderPath, err)
 		return nil
 	}
+
+	glog.Infof("Finished S2Indexer's Load() from folder %s.\n", folderPath)
 	return indexer
 }
 
 // Dump S2Indexer's content into folderPath
 func (indexer *S2Indexer) Dump(folderPath string) {
+	glog.Info("Start S2Indexer's Dump().\n")
+
 	if err := serializeS2Indexer(indexer, folderPath); err != nil {
 		glog.Errorf("Dump S2Indexer's data to folder %s failed, err=%v\n", folderPath, err)
 	}
+
+	glog.Infof("Finished S2Indexer's Dump() to folder %s.\n", folderPath)
 }
 
 // IteratePoints returns PointInfo in channel

diff --git a/integration/service/spatialindexer/s2indexer/spatial_query.go b/integration/service/spatialindexer/s2indexer/spatial_query.go
@@ -52,6 +52,14 @@ func queryNearByPoints(indexer *S2Indexer, point spatialindexer.Location, radius
 	return result
 }
 
+func generateDebugInfo4Query(point spatialindexer.Location, radius float64, cellIDs []s2.CellID) {
+	glog.Infof("During spatial_query, point = %+v, radius = %v {", point, radius)
+	for _, cellID := range cellIDs {
+		glog.Infof("%s,", cellID.ToToken())
+	}
+	glog.Info("}\n")
+}
+
 func generateDebugInfo4CellIDs(cellIDs []s2.CellID) {
 	glog.Info("=================================\n")
 	glog.Info("generateDebugInfo4CellIDs\n")