Add new randomized benchmarks (#6929)

.github/workflows/osrm-backend.yml

@@ -645,8 +645,10 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       PR_NUMBER: ${{ github.event.pull_request.number }}
       GITHUB_REPOSITORY: ${{ github.repository }}
+      RUN_BIG_BENCHMARK: ${{ contains(github.event.pull_request.labels.*.name, 'Performance') }}
     steps: 
       - name: Enable data.osm.pbf cache
+        if: ${{ ! env.RUN_BIG_BENCHMARK }}        
         uses: actions/cache@v4
         with:
           path: ~/data.osm.pbf
@@ -678,10 +680,18 @@ jobs:
           sudo apt-get update -y && sudo apt-get install ccache
       - name: Prepare data
         run: |
-          if [ ! -f "~/data.osm.pbf" ]; then
-            wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf
+          if [ "$RUN_BIG_BENCHMARK" = "true" ]; then
+              rm -rf ~/data.osm.pbf
+              wget http://download.geofabrik.de/europe/poland-latest.osm.pbf -O ~/data.osm.pbf --quiet
+              gunzip -c ./pr/test/data/poland_gps_traces.csv.gz > ~/gps_traces.csv
+          else
+              if [ ! -f "~/data.osm.pbf" ]; then
+                wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf
+              else
+                echo "Using cached data.osm.pbf"
+              fi
+              gunzip -c ./pr/test/data/berlin_gps_traces.csv.gz > ~/gps_traces.csv
           fi
-          gunzip -c ./pr/test/data/berlin_gps_traces.csv.gz > ~/gps_traces.csv
       - name: Prepare environment
         run: |
           echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV

scripts/ci/download_gps_traces.py

@@ -4,41 +4,43 @@
 import sys
 import argparse
 
-def get_osm_gps_traces(min_lon, min_lat, max_lon, max_lat):
+def get_osm_gps_traces(bboxes):
     url = 'https://api.openstreetmap.org/api/0.6/trackpoints'
     traces = []
 
     lon_step = 0.25
     lat_step = 0.25
-
-    current_min_lon = min_lon
 
-    while current_min_lon < max_lon:
-        current_max_lon = min(current_min_lon + lon_step, max_lon)
+    for bbox in bboxes:
+        min_lon, min_lat, max_lon, max_lat = map(float, bbox.split(','))
 
-        current_min_lat = min_lat
-        while current_min_lat < max_lat:
-            current_max_lat = min(current_min_lat + lat_step, max_lat)
-
-            bbox = f'{current_min_lon},{current_min_lat},{current_max_lon},{current_max_lat}'
-            print(f"Requesting bbox: {bbox}", file=sys.stderr)
-
-            params = {
-                'bbox': bbox,
-                'page': 0
-            }
-            headers = {
-                'Accept': 'application/xml'
-            }
-
-            response = requests.get(url, params=params, headers=headers)
-            if response.status_code == 200:
-                traces.append(response.content)
-            else:
-                print(f"Error fetching data for bbox {bbox}: {response.status_code} {response.text}", file=sys.stderr)
+        current_min_lon = min_lon
+        while current_min_lon < max_lon:
+            current_max_lon = min(current_min_lon + lon_step, max_lon)
 
-            current_min_lat += lat_step
-        current_min_lon += lon_step
+            current_min_lat = min_lat
+            while current_min_lat < max_lat:
+                current_max_lat = min(current_min_lat + lat_step, max_lat)
+
+                bbox_str = f'{current_min_lon},{current_min_lat},{current_max_lon},{current_max_lat}'
+                print(f"Requesting bbox: {bbox_str}", file=sys.stderr)
+
+                params = {
+                    'bbox': bbox_str,
+                    'page': 0
+                }
+                headers = {
+                    'Accept': 'application/xml'
+                }
+
+                response = requests.get(url, params=params, headers=headers)
+                if response.status_code == 200:
+                    traces.append(response.content)
+                else:
+                    print(f"Error fetching data for bbox {bbox_str}: {response.status_code} {response.text}", file=sys.stderr)
+
+                current_min_lat += lat_step
+            current_min_lon += lon_step
 
     return traces
 
@@ -68,15 +70,12 @@ def save_to_csv(data, file):
     writer.writerows(data)
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Fetch and output OSM GPS traces for a given bounding box.')
-    parser.add_argument('min_lon', type=float, help='Minimum longitude of the bounding box')
-    parser.add_argument('min_lat', type=float, help='Minimum latitude of the bounding box')
-    parser.add_argument('max_lon', type=float, help='Maximum longitude of the bounding box')
-    parser.add_argument('max_lat', type=float, help='Maximum latitude of the bounding box')
+    parser = argparse.ArgumentParser(description='Fetch and output OSM GPS traces for given bounding boxes.')
+    parser.add_argument('bboxes', nargs='+', help='Bounding boxes in the format min_lon,min_lat,max_lon,max_lat')
 
     args = parser.parse_args()
 
-    gpx_data_traces = get_osm_gps_traces(args.min_lon, args.min_lat, args.max_lon, args.max_lat)
+    gpx_data_traces = get_osm_gps_traces(args.bboxes)
     print(f"Collected {len(gpx_data_traces)} trace segments", file=sys.stderr)
 
     all_data = []

scripts/ci/run_benchmarks.sh

@@ -42,6 +42,12 @@ function run_benchmarks_for_folder {
     measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-customize $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_customize.bench"
     measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
 
+    for BENCH in nearest table trip route match; do
+        ./$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" mld ~/gps_traces.csv ${BENCH} > "$RESULTS_FOLDER/random_${BENCH}_mld.bench" || true
+        ./$BENCHMARKS_FOLDER/bench "$FOLDER/data.osrm" ch ~/gps_traces.csv ${BENCH}  > "$RESULTS_FOLDER/random_${BENCH}_ch.bench" || true
+    done
+
+
     for ALGORITHM in ch mld; do
         $BINARIES_FOLDER/osrm-routed --algorithm $ALGORITHM $FOLDER/data.osrm &
         OSRM_ROUTED_PID=$!
@@ -59,8 +65,6 @@ function run_benchmarks_for_folder {
 
         kill -9 $OSRM_ROUTED_PID
     done
-
-
 }
 
 run_benchmarks_for_folder $1 "${1}_results" $2

src/benchmarks/CMakeLists.txt

@@ -18,6 +18,7 @@ target_link_libraries(rtree-bench
 	${TBB_LIBRARIES}
 	${MAYBE_SHAPEFILE})
 
+
 add_executable(match-bench
 	EXCLUDE_FROM_ALL
 	${MatchBenchmarkSources}
@@ -35,13 +36,26 @@ add_executable(route-bench
 	route.cpp
 	$<TARGET_OBJECTS:UTIL>)
 
+
 target_link_libraries(route-bench
 	osrm
 	${BOOST_BASE_LIBRARIES}
 	${CMAKE_THREAD_LIBS_INIT}
 	${TBB_LIBRARIES}
 	${MAYBE_SHAPEFILE})
 
+add_executable(bench
+	EXCLUDE_FROM_ALL
+	bench.cpp
+	$<TARGET_OBJECTS:UTIL>)
+
+target_link_libraries(bench
+	osrm
+	${BOOST_BASE_LIBRARIES}
+	${CMAKE_THREAD_LIBS_INIT}
+	${TBB_LIBRARIES}
+	${MAYBE_SHAPEFILE})
+
 add_executable(json-render-bench
 	EXCLUDE_FROM_ALL
 	json_render.cpp
@@ -85,5 +99,6 @@ add_custom_target(benchmarks
 	packedvector-bench
 	match-bench
   route-bench
+  bench
 	json-render-bench
   alias-bench)