From 288917271f985543fc5ad3e6b98f07ba902bd83f Mon Sep 17 00:00:00 2001 From: Shankari Date: Tue, 16 Mar 2021 21:10:27 -0700 Subject: [PATCH] Support various auth policies for aggregate calls as well This fixes https://github.com/e-mission/e-mission-docs/issues/408 It is also a partial fix for https://github.com/e-mission/e-mission-docs/issues/628 We support 3 basic policies: - `no_auth`: full public access (backwards compatible behavior) - `user_only`: access only to existing users (new functionality, consistent with https://github.com/e-mission/e-mission-docs/issues/408) - `never`: disable completely Other sophisticated access control for certain users only is out of the scope at this time Testing done: - set the policy to `no_auth` - aggregate call works ``` 2021-03-16 16:25:32,859:DEBUG:123145663979520:START POST /result/metrics/timestamp 2021-03-16 16:25:32,859:DEBUG:123145663979520:Aggregate call, checking {aggregate_call_support} policy 2021-03-16 16:25:32,859:DEBUG:123145663979520:metric_list = ['duration', 'median_speed', 'count', 'distance'] 2021-03-16 16:25:32,859:DEBUG:123145663979520:['duration -> ', 'median_speed -> ', 'count -> ', 'distance -> '] 2021-03-16 16:25:32,859:DEBUG:123145663979520:for user None, returning timeseries 2021-03-16 16:25:32,867:DEBUG:123145663979520:END POST /result/metrics/timestamp 0.008590936660766602 ``` - user call works ``` 2021-03-16 16:25:32,866:DEBUG:123145669234688:START POST /result/metrics/timestamp 2021-03-16 16:25:32,867:DEBUG:123145669234688:User specific call, returning UUID 2021-03-16 16:25:32,868:DEBUG:123145669234688:methodName = skip, returning 2021-03-16 16:25:32,868:DEBUG:123145669234688:Using the skip method to verify id token REPLACEMEkVVdF9rT of length 17 2021-03-16 16:25:32,870:DEBUG:123145669234688:retUUID = cf8ccb7b-84d7-40e4-a726-7691e614b042 2021-03-16 16:25:32,876:DEBUG:123145669234688:END POST /result/metrics/timestamp cf8ccb7b-84d7-40e4-a726-7691e614b042 0.009974002838134766 ``` - switch the policy to `user_only` - user call works ``` 2021-03-16 16:25:32,866:DEBUG:123145669234688:START POST /result/metrics/timestamp 2021-03-16 16:25:32,867:DEBUG:123145669234688:User specific call, returning UUID 2021-03-16 16:25:32,868:DEBUG:123145669234688:methodName = skip, returning 2021-03-16 16:25:32,868:DEBUG:123145669234688:Using the skip method to verify id token REPLACEMEkVVdF9rT of length 17 2021-03-16 16:25:32,870:DEBUG:123145669234688:retUUID = cf8ccb7b-84d7-40e4-a726-7691e614b042 2021-03-16 16:25:32,876:DEBUG:123145669234688:END POST /result/metrics/timestamp cf8ccb7b-84d7-40e4-a726-7691e614b042 0.009974002838134766 ``` - aggregate call fails ``` 2021-03-16 16:59:25,517:DEBUG:123145504403456:START POST /result/metrics/timestamp 2021-03-16 16:59:25,517:DEBUG:123145504403456:Aggregate call, checking user_only policy 2021-03-16 16:59:25,518:DEBUG:123145504403456:END POST /result/metrics/timestamp 0.00035881996154785156 ``` with error ``` 2021-03-16 16:58:42.465 23394-23394/edu.berkeley.eecs.emission.devapp I/chromium: [INFO:CONSOLE(145)] "ERROR:Error loading aggregate data, averages not available{"status":403,"url":"http://10.0.2.2:8080/result/metrics/timestamp","headers":{"date":"Tue, 16 Mar 2021 23:59:25 GMT","content-length":"761","server":"Cheroot/8.4.2","x-android-selected-protocol":"http/1.1","x-android-response-source":"NETWORK 403","x-android-received-millis":"1615939122220","x-android-sent-millis":"1615939122206","content-type":"text/html; charset=UTF-8"},"error":"\n \n \n \n Error: 403 Forbidden\n \n \n \n

Error: 403 Forbidden

\n

Sorry, the requested URL 'http://10.0.2.2:8080/result/metrics/timestamp'\n caused an error:

\n 
aggregations only available to users
\n \n \n"}", source: http://localhost/_app_file_/data/user/0/edu.berkeley.eecs.emission.devapp/files/phonegapdevapp/www/index.html (145) ``` - switch the policy to `never`, fails with error ``` 2021-03-16 17:13:20.422 23394-23394/edu.berkeley.eecs.emission.devapp I/chromium: [INFO:CONSOLE(145)] "Error loading aggregate data, averages not available{"status":404,"url":"http://10.0.2.2:8080/result/metrics/timestamp","headers":{"date":"Wed, 17 Mar 2021 00:14:03 GMT","content-length":"754","server":"Cheroot/8.4.2","x-android-selected-protocol":"http/1.1","x-android-response-source":"NETWORK 404","x-android-received-millis":"1615940000171","x-android-sent-millis":"1615940000159","content-type":"text/html; charset=UTF-8"},"error":"\n \n \n \n Error: 404 Not Found\n \n \n \n

Error: 404 Not Found

\n

Sorry, the requested URL 'http://10.0.2.2:8080/result/metrics/timestamp'\n caused an error:

\n 
Aggregate calls not supported
\n \n \n"}", source: http://localhost/_app_file_/data/user/0/edu.berkeley.eecs.emission.devapp/files/phonegapdevapp/www/index.html (145) ``` - switch the policy to an invalid valid, fails with error ``` 2021-03-16 17:14:25.561 23394-23394/edu.berkeley.eecs.emission.devapp I/chromium: [INFO:CONSOLE(145)] "ERROR:Error loading aggregate data, averages not available{"status":500,"url":"http://10.0.2.2:8080/result/metrics/timestamp","headers":{"date":"Wed, 17 Mar 2021 00:15:08 GMT","content-length":"1550","server":"Cheroot/8.4.2","x-android-selected-protocol":"http/1.1","x-android-response-source":"NETWORK 500","x-android-received-millis":"1615940065310","x-android-sent-millis":"1615940065297","content-type":"text/html; charset=UTF-8"},"error":"\n \n \n \n Error: 500 Internal Server Error\n \n \n \n

Error: 500 Internal Server Error

\n

Sorry, the requested URL 'http://10.0.2.2:8080/result/metrics/timestamp'\n caused an error:

\n 
Internal Server Error
\n

Exception:

\n 
KeyError('foobar')
\n

Traceback:

\n 
Traceback (most recent call last):\n  File "/Users/kshankar/e-mission/e-mission-server/emission/net/api/bottle.py", line 997, in _handle\n    out = route.call(**args)\n  File "/Users/kshankar/e-mission/e-mission-server/emission/net/api/bottle.py", line 1998, in wrapper\n    rv = callback(*a, **ka)\n  File "emission/net/api/cfc_webapp.py", line 466, in summarize_metrics\n    user_uuid = get_user_or_aggregate_auth(request)\n  File "emission/net/api/cfc_webapp.py", line 621, in get_user_or_aggregate_auth\n    return aggregate_call_map[aggregate_call_support](request)\nKeyError: 'foobar'\n
\n \n \n"}", source: http://localhost/_app_file_/data/user/0/edu.berkeley.eecs.emission.devapp/files/phonegapdevapp/www/index.html (145) ``` - changed the phone code to send a user token for aggregate calls as well, worked ``` 2021-03-16 18:52:47,214:DEBUG:123145648730112:START POST /result/metrics/timestamp 2021-03-16 18:52:47,214:DEBUG:123145648730112:Aggregate call, checking user_only policy 2021-03-16 18:52:47,214:DEBUG:123145648730112:methodName = skip, returning 2021-03-16 18:52:47,215:DEBUG:123145648730112:Using the skip method to verify id token REPLACEMEkVVdF9rT of length 17 2021-03-16 18:52:47,216:DEBUG:123145648730112:retUUID = cf8ccb7b-84d7-40e4-a726-7691e614b042 2021-03-16 18:52:47,223:DEBUG:123145648730112:END POST /result/metrics/timestamp cf8ccb7b-84d7-40e4-a726-7691e614b042 0.009236335754394531 ``` --- conf/net/api/webserver.conf.sample | 4 ++- emission/net/api/cfc_webapp.py | 55 +++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/conf/net/api/webserver.conf.sample b/conf/net/api/webserver.conf.sample index 01db7dc18..819a12976 100644 --- a/conf/net/api/webserver.conf.sample +++ b/conf/net/api/webserver.conf.sample @@ -11,6 +11,8 @@ "port" : "8080", "__comment": "1 hour = 60 min = 60 * 60 sec", "timeout" : "3600", - "auth": "skip" + "auth": "skip", + "__comment": "Options are no_auth, user_only, never", + "aggregate_call_auth": "no_auth" } } diff --git a/emission/net/api/cfc_webapp.py b/emission/net/api/cfc_webapp.py index 4803461e1..179643afc 100644 --- a/emission/net/api/cfc_webapp.py +++ b/emission/net/api/cfc_webapp.py @@ -67,6 +67,7 @@ socket_timeout = config_data["server"]["timeout"] log_base_dir = config_data["paths"]["log_base_dir"] auth_method = config_data["server"]["auth"] +aggregate_call_auth = config_data["server"]["aggregate_call_auth"] BaseRequest.MEMFILE_MAX = 1024 * 1024 * 1024 # Allow the request size to be 1G # to accomodate large section sizes @@ -132,12 +133,21 @@ def server_templates(filepath): logging.debug("static filepath = %s" % filepath) return static_file(filepath, "%s/%s" % (static_path, "templates")) +# Backward compat to handle older clients +# Remove in 2023 after everybody has upgraded +# We used to use the presence or absence of the "user" field +# to determine whether this was an aggregate call or not +# now we expect the client to fill it in +def _fill_aggregate_backward_compat(request): + if 'aggregate' not in request.json: + # Aggregate if there is no user + # no aggregate if there is a user + request.json["aggregate"] = ('user' not in request.json) + @post("/result/heatmap/pop.route/") def getPopRoute(time_type): - if 'user' in request.json: - user_uuid = getUUID(request) - else: - user_uuid = None + _fill_aggregate_backward_compat(request) + user_uuid = get_user_or_aggregate_auth(request) if 'from_local_date' in request.json and 'to_local_date' in request.json: start_time = request.json['from_local_date'] @@ -160,10 +170,8 @@ def getPopRoute(time_type): @post("/result/heatmap/incidents/") def getStressMap(time_type): - if 'user' in request.json: - user_uuid = getUUID(request) - else: - user_uuid = None + _fill_aggregate_backward_compat(request) + user_uuid = get_user_or_aggregate_auth(request) # modes = request.json['modes'] # hardcode modes to None because we currently don't store @@ -330,10 +338,9 @@ def getUserProfile(): @post('/result/metrics/') def summarize_metrics(time_type): - if 'user' in request.json: - user_uuid = getUUID(request) - else: - user_uuid = None + _fill_aggregate_backward_compat(request) + user_uuid = get_user_or_aggregate_auth(request) + start_time = request.json['start_time'] end_time = request.json['end_time'] freq_name = request.json['freq'] @@ -465,6 +472,30 @@ def after_request(): # This should only be used by createUserProfile since we may not have a UUID # yet. All others should use the UUID. +def _get_uuid_bool_wrapper(request): + try: + getUUID(request) + return True + except: + return False + +def get_user_or_aggregate_auth(request): + # If this is not an aggregate call, returns the uuid + # If this is an aggregate call, returns None if the call is valid, otherwise aborts + # we only support aggregates on a subset of calls, so we don't need the + # `inHeader` parameter to `getUUID` + aggregate_call_map = { + "no_auth": lambda r: None, + "user_only": lambda r: None if _get_uuid_bool_wrapper(request) else abort(403, "aggregations only available to users"), + "never": lambda r: abort(404, "Aggregate calls not supported") + } + if request.json["aggregate"] == False: + logging.debug("User specific call, returning UUID") + return getUUID(request) + else: + logging.debug(f"Aggregate call, checking {aggregate_call_auth} policy") + return aggregate_call_map[aggregate_call_auth](request) + def getUUID(request, inHeader=False): try: retUUID = enaa.getUUID(request, auth_method, inHeader)