Merge branch 'master' into react-select

backend/models/gtfs.py

@@ -876,8 +876,10 @@ def get_route_data(self, route):
             title = f'{short_name}-{long_name}'
         elif isinstance(short_name, str):
             title = short_name
-        else:
+        elif isinstance(long_name, str):
             title = long_name
+        else:
+            title = gtfs_route_id
 
         type = int(route.route_type) if hasattr(route, 'route_type') else None
         url = route.route_url if hasattr(route, 'route_url') and isinstance(route.route_url, str) else None
@@ -990,26 +992,6 @@ def save_routes(self, save_to_s3=True):
 
         routes_data = sorted(routes_data, key=sort_key)
 
-        data_str = json.dumps({
-            'version': routeconfig.DefaultVersion,
-            'routes': routes_data
-        }, separators=(',', ':'))
-
-        cache_path = routeconfig.get_cache_path(agency_id)
+        routes = [routeconfig.RouteConfig(agency_id, route_data) for route_data in routes_data]
 
-        with open(cache_path, "w") as f:
-            f.write(data_str)
-
-        if save_to_s3:
-            s3 = boto3.resource('s3')
-            s3_path = routeconfig.get_s3_path(agency_id)
-            s3_bucket = config.s3_bucket
-            print(f'saving to s3://{s3_bucket}/{s3_path}')
-            object = s3.Object(s3_bucket, s3_path)
-            object.put(
-                Body=gzip.compress(bytes(data_str, 'utf-8')),
-                CacheControl='max-age=86400',
-                ContentType='application/json',
-                ContentEncoding='gzip',
-                ACL='public-read'
-            )
+        routeconfig.save_routes(agency_id, routes, save_to_s3=save_to_s3)

backend/models/routeconfig.py

@@ -1,4 +1,4 @@
-import re, os, time, requests, json
+import re, os, time, requests, json, boto3, gzip
 from . import util, config
 
 DefaultVersion = 'v3a'
@@ -148,4 +148,29 @@ def get_route_config(agency_id, route_id, version=DefaultVersion):
     for route in get_route_list(agency_id, version):
         if route.id == route_id:
             return route
-    return None
+    return None
+
+def save_routes(agency_id, routes, save_to_s3=False):
+    data_str = json.dumps({
+        'version': DefaultVersion,
+        'routes': [route.data for route in routes]
+    }, separators=(',', ':'))
+
+    cache_path = get_cache_path(agency_id)
+
+    with open(cache_path, "w") as f:
+        f.write(data_str)
+
+    if save_to_s3:
+        s3 = boto3.resource('s3')
+        s3_path = get_s3_path(agency_id)
+        s3_bucket = config.s3_bucket
+        print(f'saving to s3://{s3_bucket}/{s3_path}')
+        object = s3.Object(s3_bucket, s3_path)
+        object.put(
+            Body=gzip.compress(bytes(data_str, 'utf-8')),
+            CacheControl='max-age=86400',
+            ContentType='application/json',
+            ContentEncoding='gzip',
+            ACL='public-read'
+        )

backend/models/trip_times.py

@@ -26,7 +26,7 @@ def get_completed_trip_times(
     # or by departure / arrival time (if is_loop is true).
 
     if (len(s1_trip_values) == 0) or (len(s2_trip_values) == 0):
-        return []
+        return np.array([])
 
     if is_loop:
         # If s1 and s2 are the same stop, this will compute the time to complete 1 full loop

backend/tests/test_schema.py

@@ -4,13 +4,15 @@
 from graphene.test import Client
 import pandas as pd
 import datetime
-from backend.models import arrival_history
+from backend.models import arrival_history, routeconfig
 from backend.models.schema import metrics_api
 
 class SchemaTest(unittest.TestCase):
 
     def test_route_metrics_query(self):
 
+        routeconfig.save_routes('test', [], save_to_s3=False)
+
         d = datetime.date(2019,12,28)
         start_time = 1577530800
         end_time = 1577617200

backend/tests/test_trip_times.py

@@ -0,0 +1,94 @@
+import backend_path
+import unittest
+import datetime
+import numpy as np
+import pandas as pd
+from backend.models import trip_times
+
+class TripTimesTest(unittest.TestCase):
+
+    def test_trip_times(self):
+
+        trips1 = np.array([1,    2,    3,            5,            7,     9,     10])
+        times1 = np.array([6000, 9000, 12000,        18000,        30000, 60000, 90000])
+        trips2 = np.array([1,          3,     4,     5,     6,     7,     9])
+        times2 = np.array([6300,       12600, 15000, 18900, 20000, 30510, 66000])
+                           # 5m        # 10m         # 15m         # 8.5m   # 100m
+
+        order1 = np.arange(len(trips1))
+        np.random.shuffle(order1)
+
+        order2 = np.arange(len(trips2))
+        np.random.shuffle(order2)
+
+        trips = trip_times.get_completed_trip_times(
+            trips1[order1], times1[order1], trips2[order2], times2[order2],
+            is_loop=False
+        ).tolist()
+
+        self.assertEqual(trips, [5, 10, 15, 8.5, 100])
+
+        trip_min, arrival_times = trip_times.get_matching_trips_and_arrival_times(
+            trips1[order1], times1[order1], trips2[order2], times2[order2],
+            is_loop=False
+        )
+
+        inverse_order = np.argsort(order1)
+
+        np.testing.assert_equal(trip_min[inverse_order], [5, np.nan, 10, 15, 8.5, 100, np.nan])
+        np.testing.assert_equal(arrival_times[inverse_order], [ 6300, np.nan, 12600, 18900, 30510, 66000, np.nan])
+
+        trips = trip_times.get_completed_trip_times(
+            np.array([1]),
+            np.array([6000]),
+            np.array([99]),
+            np.array([166000]),
+            is_loop=False
+        ).tolist()
+
+        self.assertEqual(trips, [])
+
+        trips = trip_times.get_completed_trip_times(
+            np.array([]),
+            np.array([]),
+            np.array([]),
+            np.array([]),
+            is_loop=False
+        ).tolist()
+
+        self.assertEqual(trips, [])
+
+        # test loop routes
+
+        trips1 = np.array([1,    2,     1,            2,            1,     2,     1])
+        times1 = np.array([6000, 9000,  12000,        18000,        30000, 60000, 90000])
+        trips2 = np.array([1,    2,     1,     3,     2,     2,     1,     2])
+        times2 = np.array([6300, 12600, 15000, 18900, 21090, 30480, 36000, 57000])
+                           # 5m  # 60m   # 50m        # 51.5m       # 100m
+
+        order1 = np.arange(len(trips1))
+        np.random.shuffle(order1)
+
+        order2 = np.arange(len(trips2))
+        np.random.shuffle(order2)
+
+        trips = trip_times.get_completed_trip_times(
+            trips1[order1], times1[order1], trips2[order2], times2[order2],
+            is_loop=True
+        ).tolist()
+
+        self.assertEqual(trips, [5,60,50,51.5,100])
+
+        trip_min, arrival_times = trip_times.get_matching_trips_and_arrival_times(
+            trips1[order1], times1[order1], trips2[order2], times2[order2],
+            is_loop=True
+        )
+
+        inverse_order = np.argsort(order1)
+
+        np.testing.assert_equal(trip_min[inverse_order], [5, 60, 50, 51.5, 100, np.nan, np.nan])
+        np.testing.assert_equal(arrival_times[inverse_order], [ 6300, 12600, 15000, 21090, 36000, np.nan, np.nan])
+
+
+if __name__ == '__main__':
+    unittest.main()

backend/tests/test_wait_times.py

@@ -0,0 +1,143 @@
+import backend_path
+import unittest
+import datetime
+import numpy as np
+import pandas as pd
+from backend.models import wait_times
+
+class WaitTimesTest(unittest.TestCase):
+
+    def test_get_stats(self):
+
+        time_values = np.array([
+            300,
+            600, # 300s = 5m
+            900, # 300s = 5m
+            1500, # 600s = 10m
+            3600, # 2100s = 35m
+            7200 # 3600 = 60m
+        ]) # total interval: 115m
+
+        stats = wait_times.get_stats(time_values)
+
+        avg_wait_time = (5*5/2 + 5*5/2 + 10*10/2 + 35*35/2 + 60*60/2)/115
+
+        self.assertEqual(stats.get_average(), avg_wait_time)
+
+        prob_lt_5 = (5*5)/115
+        self.assertEqual(stats.get_quantile(prob_lt_5), 5)
+        self.assertEqual(stats.get_probability_less_than(5), prob_lt_5)
+        self.assertEqual(stats.get_probability_greater_than(5), 1-prob_lt_5)
+
+        prob_lt_10 = ((5*5)+(5*3))/115
+        self.assertEqual(stats.get_quantile(prob_lt_10), 10)
+        self.assertEqual(stats.get_probability_less_than(10), prob_lt_10)
+
+        prob_lt_35 = ((5*5)+(5*3)+(25*2))/115
+        self.assertEqual(stats.get_quantile(prob_lt_35), 35)
+        self.assertEqual(stats.get_probability_less_than(35), prob_lt_35)
+
+        self.assertEqual(stats.get_probability_less_than(18.75), 0.5)
+        self.assertEqual(stats.get_probability_less_than(100), 1)
+        self.assertEqual(stats.get_probability_less_than(-1), 0)
+
+        histogram = stats.get_histogram([0,5,10,15,30,60,90])
+
+        self.assertEqual(len(histogram), 6)
+        self.assertAlmostEqual(histogram[0], prob_lt_5)
+        self.assertAlmostEqual(histogram[1], prob_lt_10-prob_lt_5)
+        self.assertAlmostEqual(histogram[2], (prob_lt_35-prob_lt_10)/5,)
+        self.assertAlmostEqual(histogram[3], (prob_lt_35-prob_lt_10)*3/5)
+        self.assertAlmostEqual(histogram[4], 1-np.sum(histogram[0:4]))
+        self.assertEqual(0, histogram[5])
+
+        quantiles = stats.get_quantiles([0, 0.1, 0.5, 0.9, 1])
+
+        self.assertEqual(len(quantiles), 5)
+        self.assertEqual(quantiles[0], 0)
+        self.assertAlmostEqual(quantiles[1], 2.3)
+        self.assertEqual(quantiles[2], 18.75)
+        self.assertEqual(quantiles[3], 48.5)
+        self.assertEqual(quantiles[4], 60)
+
+        self.assertEqual(stats.get_percentiles([50,90]).tolist(), [18.75, 48.5])
+        self.assertEqual(stats.get_percentile(100), 60)
+
+        sampled_waits = stats.get_sampled_waits(60)
+        self.assertEqual(len(sampled_waits), 115)
+        self.assertEqual(np.min(sampled_waits), 0)
+        self.assertAlmostEqual(np.average(sampled_waits), 21.1, places=1)
+        self.assertAlmostEqual(np.median(sampled_waits), 18.0, places=1)
+        self.assertEqual(np.max(sampled_waits), 59)
+
+        # test arrival after end of interval
+
+        stats = wait_times.get_stats(time_values, 600, 1020) # 7 minutes long
+        self.assertAlmostEqual(stats.get_average(), ((5*5)/2 + (2*2)/2 + (2*8))/7)
+
+        self.assertEqual(stats.get_quantiles([0,1]).tolist(), [0,10])
+
+        self.assertAlmostEqual(stats.get_probability_less_than(4), 4/7)
+        self.assertAlmostEqual(stats.get_probability_less_than(5), 5/7)
+        self.assertAlmostEqual(stats.get_probability_less_than(8), 5/7)
+        self.assertAlmostEqual(stats.get_probability_less_than(9), 6/7)
+
+        # test no arrivals in interval
+        stats = wait_times.get_stats(np.array([]))
+        self.assertEqual(stats.get_average(), None)
+        self.assertEqual(stats.get_percentile(50), None)
+        self.assertEqual(stats.get_percentiles([50]), None)
+        self.assertEqual(stats.get_probability_less_than(5), None)
+        self.assertEqual(stats.get_probability_greater_than(5), None)
+        self.assertEqual(stats.get_quantile(0), None)
+        self.assertEqual(stats.get_quantiles([0,0.5]), None)
+        self.assertEqual(stats.get_histogram([0,60]), None)
+        self.assertEqual(stats.get_sampled_waits(), None)
+
+    def test_combine_stats(self):
+
+        stats1 = wait_times.get_stats(np.array([
+            300,
+            600,
+            900,
+            1500,
+            3600,
+            7200
+        ])) # 115 min
+
+        stats2 = wait_times.get_stats(np.array([
+            30300,
+            30360,
+            31800,
+            32100,
+            33000,
+            36000,
+            39360,
+        ])) # 151 min
+
+        stats3 = wait_times.get_stats(np.array([]))
+
+        stats4 = wait_times.get_stats(np.array([
+            60300,
+            63000,
+            69600,
+        ])) # 155 min
+
+        combined = wait_times.combine_stats([stats1, stats2, stats3, stats4])
+
+        self.assertAlmostEqual(combined.get_average(), (stats1.get_average() + stats2.get_average() + stats4.get_average()) / 3, places=3)
+        self.assertAlmostEqual(combined.get_percentile(50), 23.664, places=3)
+        self.assertEqual(combined.get_percentiles([0,100]).tolist(), [0,110])
+        self.assertAlmostEqual(combined.get_probability_less_than(5), 0.151, places=3)
+        self.assertAlmostEqual(combined.get_probability_greater_than(5), 1-0.151, places=3)
+        self.assertAlmostEqual(combined.get_quantile(0.5), 23.664, places=3)
+        self.assertEqual(combined.get_quantiles([0,1]).tolist(), [0,110])
+
+        histogram = combined.get_histogram([0,30,60,110])
+        self.assertAlmostEqual(histogram[0], 0.593, places=3)
+        self.assertAlmostEqual(histogram[1], 0.300, places=3)
+        self.assertAlmostEqual(histogram[2], 0.108, places=3)
+        self.assertEqual(len(combined.get_sampled_waits()), 421)
+
+if __name__ == '__main__':
+    unittest.main()

frontend/src/App.jsx

@@ -1,4 +1,4 @@
-import React, { Fragment } from 'react';
+import React from 'react';
 import { connect } from 'react-redux';
 
 import './App.css';
@@ -10,17 +10,19 @@ import RouteScreen from './screens/RouteScreen';
 import DataDiagnostic from './screens/DataDiagnostic';
 import Isochrone from './screens/Isochrone';
 
+const Components = {
+  About,
+  Isochrone,
+  Landing,
+  Dashboard,
+  RouteScreen,
+  DataDiagnostic,
+  NotFound
+};
+
 const App = ({ page }) => {
-  const components = {
-    About: <About />,
-    Isochrone: <Isochrone />,
-    Landing: <Landing />,
-    Dashboard: <Dashboard />,
-    Route: <RouteScreen />,
-    DataDiagnostic: <DataDiagnostic />,
-    NotFound: <NotFound />,
-  };
-  return <Fragment>{components[page]}</Fragment>;
+  const Component = Components[page];
+  return <Component />;
 };
 
 const mapStateToProps = ({ page }) => ({ page });