From 86580aefeae07bb141357d664444f3e427e2cb5e Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Mon, 1 Sep 2014 21:08:23 -0700 Subject: [PATCH 01/11] adding the ability to run dframe_explorer from geopandas basically we monkey patch geopandas to have an explore method that calls dframe_explorer with all the right configuration. This is nice because geopandas * can convert to 4326 * has the center of the shapefile already * can read shapefiles rather than geojson into a leaflet map * can convert to geojson on the fly so that no file on the filesystem is required This also calls the webbrowser automatically when everything is ready --- urbansim/__init__.py | 3 ++ urbansim/geopandaspatch.py | 55 ++++++++++++++++++++++++++++++ urbansim/maps/dframe_explorer.html | 8 +++++ urbansim/maps/dframe_explorer.py | 28 ++++++++++++--- 4 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 urbansim/geopandaspatch.py diff --git a/urbansim/__init__.py b/urbansim/__init__.py index d7dc9b38..d1e03138 100644 --- a/urbansim/__init__.py +++ b/urbansim/__init__.py @@ -2,3 +2,6 @@ from .patsypatch import patch_patsy patch_patsy() + +from .geopandaspatch import patch_geopandas +patch_geopandas() diff --git a/urbansim/geopandaspatch.py b/urbansim/geopandaspatch.py new file mode 100644 index 00000000..acbbdf26 --- /dev/null +++ b/urbansim/geopandaspatch.py @@ -0,0 +1,55 @@ +from urbansim.maps import dframe_explorer +import pandas as pd + + +def explore(self, + dataframe_d=None, + center=None, + zoom=11, + geom_name=None, # from JSON file, use index if None + join_name='zone_id', # from data frames + precision=2, + port=8765, + host='localhost', + testing=False): + + if dataframe_d is None: + dataframe_d = {} + + # add the geodataframe + df = pd.DataFrame(self) + if geom_name is None: + df[join_name] = df.index + dataframe_d["local"] = df + + # need to check if it's already 4326 + if self.crs != 4326: + self = self.to_crs(epsg=4326) + + bbox = self.total_bounds + if center is None: + center = [(bbox[1]+bbox[3])/2, (bbox[0]+bbox[2])/2] + + self.to_json() + + dframe_explorer.start( + dataframe_d, + center=center, + zoom=zoom, + shape_json=self.to_json(), + geom_name=geom_name, # from JSON file + join_name=join_name, # from data frames + precision=precision, + port=port, + host=host, + testing=testing + ) + + +def patch_geopandas(): + """ + Add a new function to the geodataframe called explore which uses the + urbansim function dataframe_explorer. + """ + import geopandas + geopandas.GeoDataFrame.explore = explore diff --git a/urbansim/maps/dframe_explorer.html b/urbansim/maps/dframe_explorer.html index b1f2bd60..2625f9f3 100644 --- a/urbansim/maps/dframe_explorer.html +++ b/urbansim/maps/dframe_explorer.html @@ -200,7 +200,11 @@ function style_f(feature) { if(data && q) { + {% if geom_name %} var val = data[feature.properties[GEOMNAME]]; + {% else %} + var val = data[feature.id]; + {% endif %} var v = q(val); } fo = .7; @@ -225,7 +229,11 @@ shapeLayer = new L.geoJson(zones, { style: style_f, onEachFeature: function (feature, layer) { + {% if geom_name %} feature_d[feature.properties[GEOMNAME]] = layer; + {% else %} + feature_d[feature.id] = layer; + {% endif %} //layer.bindPopup("No value"); } }); diff --git a/urbansim/maps/dframe_explorer.py b/urbansim/maps/dframe_explorer.py index a8fd0be8..38296131 100644 --- a/urbansim/maps/dframe_explorer.py +++ b/urbansim/maps/dframe_explorer.py @@ -4,6 +4,8 @@ import numpy as np import pandas as pd import os +import json +import webbrowser from jinja2 import Environment @@ -60,6 +62,8 @@ def index(): @route('/data/') def data_static(filename): + if filename == "internal": + return SHAPES return static_file(filename, root='./data') @@ -89,10 +93,12 @@ def start(views, zoom : int The initial zoom level of the map shape_json : str - The path to the geojson file which contains that shapes that will be - displayed + Can either be the geojson itself or the path to a file which contains + the geojson that describes the shapes to display (uses os.path.exists + to check for a file on the filesystem) geom_name : str - The field name from the JSON file which contains the id of the geometry + The field name from the JSON file which contains the id of the + geometry - if it's None, use the id of the geojson feature join_name : str The column name from the dataframes passed as views (must be in each view) which joins to geom_name in the shapes @@ -111,9 +117,20 @@ def start(views, queries from a web browser """ - global DFRAMES, CONFIG + global DFRAMES, CONFIG, SHAPES DFRAMES = {str(k): views[k] for k in views} + print shape_json + if not testing and not os.path.exists(shape_json): + # if the file doesn't exist, we try to use it as json + try: + json.loads(shape_json) + except: + assert 0, "The json passed in appears to be neither a parsable " \ + "json format nor a file that exists on the file system" + SHAPES = shape_json + shape_json = "data/internal" + config = { 'center': str(center), 'zoom': zoom, @@ -135,4 +152,7 @@ def start(views, if testing: return + # open in a new tab, if possible + webbrowser.open("http://%s:%s" % (host, port), new=2) + run(host=host, port=port, debug=True) From 22d54cd5f972aba6a24fd43e0d1ef4dc92fa49cc Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Mon, 1 Sep 2014 21:14:53 -0700 Subject: [PATCH 02/11] geopandas is a dependency of this branch --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0d8701b1..b746bcb5 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ packages=find_packages(exclude=['*.tests']), install_requires=[ 'bottle>=0.12.5', + 'geopandas>=.1', 'matplotlib>=1.3.1', 'numpy>=1.8.0', 'pandas>=0.13.1', From 7ef628cac3f4ccb5a213afa1a4a2544ac1a68397 Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Mon, 1 Sep 2014 21:17:32 -0700 Subject: [PATCH 03/11] named the version of geopandas incorrectly --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b746bcb5..12e443b2 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ packages=find_packages(exclude=['*.tests']), install_requires=[ 'bottle>=0.12.5', - 'geopandas>=.1', + 'geopandas>=0.1.0', 'matplotlib>=1.3.1', 'numpy>=1.8.0', 'pandas>=0.13.1', From 8ea9c22bcdd18128f99021cbf161b7dc0f48bbd1 Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Tue, 2 Sep 2014 14:56:49 -0700 Subject: [PATCH 04/11] this removes the monkey patching for geocanvas and moves it to dframe_explorer still needs a test for this new function --- .travis.yml | 2 +- setup.py | 4 +-- urbansim/__init__.py | 3 -- urbansim/geopandaspatch.py | 55 -------------------------------- urbansim/maps/dframe_explorer.py | 52 ++++++++++++++++++++++++++++++ 5 files changed, 55 insertions(+), 61 deletions(-) delete mode 100644 urbansim/geopandaspatch.py diff --git a/.travis.yml b/.travis.yml index 8d0434fe..b9459f56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ install: - sudo conda init - sudo conda update --yes conda - | - conda create -p $HOME/py --yes ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pandana pytables pytest pyyaml toolz "python=$TRAVIS_PYTHON_VERSION" -c "synthicity" + conda create -p $HOME/py --yes fiona ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pandana pytables pytest pyyaml shapely toolz "python=$TRAVIS_PYTHON_VERSION" -c "synthicity" - export PATH=$HOME/py/bin:$PATH - pip install simplejson bottle - pip install pytest-cov coveralls pep8 diff --git a/setup.py b/setup.py index 12e443b2..c17e92f5 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ packages=find_packages(exclude=['*.tests']), install_requires=[ 'bottle>=0.12.5', - 'geopandas>=0.1.0', 'matplotlib>=1.3.1', 'numpy>=1.8.0', 'pandas>=0.13.1', @@ -46,6 +45,7 @@ 'toolz>=0.7.0' ], extras_require = { - 'pandana': ['pandana>=0.1'] + 'pandana': ['pandana>=0.1'], + 'geopandas': ['geopandas>=0.1.0'] } ) diff --git a/urbansim/__init__.py b/urbansim/__init__.py index d1e03138..d7dc9b38 100644 --- a/urbansim/__init__.py +++ b/urbansim/__init__.py @@ -2,6 +2,3 @@ from .patsypatch import patch_patsy patch_patsy() - -from .geopandaspatch import patch_geopandas -patch_geopandas() diff --git a/urbansim/geopandaspatch.py b/urbansim/geopandaspatch.py deleted file mode 100644 index acbbdf26..00000000 --- a/urbansim/geopandaspatch.py +++ /dev/null @@ -1,55 +0,0 @@ -from urbansim.maps import dframe_explorer -import pandas as pd - - -def explore(self, - dataframe_d=None, - center=None, - zoom=11, - geom_name=None, # from JSON file, use index if None - join_name='zone_id', # from data frames - precision=2, - port=8765, - host='localhost', - testing=False): - - if dataframe_d is None: - dataframe_d = {} - - # add the geodataframe - df = pd.DataFrame(self) - if geom_name is None: - df[join_name] = df.index - dataframe_d["local"] = df - - # need to check if it's already 4326 - if self.crs != 4326: - self = self.to_crs(epsg=4326) - - bbox = self.total_bounds - if center is None: - center = [(bbox[1]+bbox[3])/2, (bbox[0]+bbox[2])/2] - - self.to_json() - - dframe_explorer.start( - dataframe_d, - center=center, - zoom=zoom, - shape_json=self.to_json(), - geom_name=geom_name, # from JSON file - join_name=join_name, # from data frames - precision=precision, - port=port, - host=host, - testing=testing - ) - - -def patch_geopandas(): - """ - Add a new function to the geodataframe called explore which uses the - urbansim function dataframe_explorer. - """ - import geopandas - geopandas.GeoDataFrame.explore = explore diff --git a/urbansim/maps/dframe_explorer.py b/urbansim/maps/dframe_explorer.py index 38296131..40a273e9 100644 --- a/urbansim/maps/dframe_explorer.py +++ b/urbansim/maps/dframe_explorer.py @@ -156,3 +156,55 @@ def start(views, webbrowser.open("http://%s:%s" % (host, port), new=2) run(host=host, port=port, debug=True) + + +def geodataframe_explore(gdf, + dataframe_d=None, + center=None, + zoom=11, + geom_name=None, # from JSON file, use index if None + join_name='zone_id', # from data frames + precision=2, + port=8765, + host='localhost', + testing=False): + """ + This method + """ + try: + import geopandas + except: + raise ImportError("This method requires that geopandas be installed " + "in order to work correctly") + + if dataframe_d is None: + dataframe_d = {} + + # add the geodataframe + df = pd.DataFrame(gdf) + if geom_name is None: + df[join_name] = df.index + dataframe_d["local"] = df + + # need to check if it's already 4326 + if gdf.crs != 4326: + self = gdf.to_crs(epsg=4326) + + bbox = self.total_bounds + if center is None: + center = [(bbox[1]+bbox[3])/2, (bbox[0]+bbox[2])/2] + + self.to_json() + + start( + dataframe_d, + center=center, + zoom=zoom, + shape_json=self.to_json(), + geom_name=geom_name, # from JSON file + join_name=join_name, # from data frames + precision=precision, + port=port, + host=host, + testing=testing + ) From 25f8fcfd669f1f21cf4bd2aafc31fb5ecb2879ee Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Tue, 2 Sep 2014 15:54:16 -0700 Subject: [PATCH 05/11] adding very small test for geodataframe_explorer --- urbansim/maps/dframe_explorer.py | 10 +++++++++- urbansim/maps/tests/test.geojson | 1 + urbansim/maps/tests/test_explorer.py | 21 +++++++++++++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 urbansim/maps/tests/test.geojson diff --git a/urbansim/maps/dframe_explorer.py b/urbansim/maps/dframe_explorer.py index 40a273e9..8d5a6653 100644 --- a/urbansim/maps/dframe_explorer.py +++ b/urbansim/maps/dframe_explorer.py @@ -169,7 +169,15 @@ def geodataframe_explore(gdf, host='localhost', testing=False): """ - This method + This method wraps the start method above. The parameters are the same as + above but many are optional and the defaults can be derived from the + dataframe in the following way. + + If you don't pass a dataframe_d, only the fields directly on the + geodataframe will be available. The center will be derived from the + center of the dataframe's bounding box. The geom_name is optional and if + it is not set or set to None, the index of the geodataframe will be used + for joining attributes to shapes. """ try: import geopandas diff --git a/urbansim/maps/tests/test.geojson b/urbansim/maps/tests/test.geojson new file mode 100644 index 00000000..417ff743 --- /dev/null +++ b/urbansim/maps/tests/test.geojson @@ -0,0 +1 @@ +{"type": "FeatureCollection", "features": [{"geometry": {"type": "Polygon", "coordinates": [[[-122.38925866249373, 37.76310903745636], [-122.38927146704587, 37.7632459782484], [-122.38891986285451, 37.76326717960533], [-122.38890672864976, 37.76313025861109], [-122.38925866249373, 37.76310903745636]]]}, "type": "Feature", "id": "92050", "properties": {"new_residential_units": 23, "max_height_per_zoning": 68.0, "max_far_per_zoning": 5.0, "acquisition_cost": "4,060,000", "current_use": "Flat & Store", "current_building_sqft": "3,400", "new_building_sqft": "25,000", "google_maps": "Click.", "average_sales_price_sqft": 932, "parcel_size": "5,000", "address": "2092 3RD ST", "new_building_revenue": "16,620,000", "new_building_cost": "7,570,000", "fill": "yellow"}}]} diff --git a/urbansim/maps/tests/test_explorer.py b/urbansim/maps/tests/test_explorer.py index 81b04b8a..d1b47d5f 100644 --- a/urbansim/maps/tests/test_explorer.py +++ b/urbansim/maps/tests/test_explorer.py @@ -1,5 +1,6 @@ import pytest import pandas as pd +import geopandas as gpd from .. import dframe_explorer @@ -12,6 +13,11 @@ def simple_map_input(): index=['a', 'b', 'c']) +@pytest.fixture +def simple_geojson(): + return gpd.read_file("test.geojson") + + def test_explorer(simple_map_input): dframe_explorer.enable_cors() dframe_explorer.ans_options() @@ -20,9 +26,11 @@ def test_explorer(simple_map_input): d = {"dfname": simple_map_input} dframe_explorer.start(d, testing=True) - dframe_explorer.map_query("dfname", "empty", "zone_id", "test_var", "mean()") + dframe_explorer.map_query("dfname", "empty", "zone_id", "test_var", + "mean()") - dframe_explorer.map_query("dfname", "empty", "zone_id", "test_var > 1", "mean()") + dframe_explorer.map_query("dfname", "empty", "zone_id", "test_var > 1", + "mean()") dframe_explorer.index() @@ -33,3 +41,12 @@ def test_explorer(simple_map_input): with pytest.raises(Exception): dframe_explorer.start(d, testing=True) + + +def test_geodf_explorer(simple_map_input, simple_geojson): + + d = {"dfname": simple_map_input} + dframe_explorer.geodataframe_explore(simple_geojson, dataframe_d=d, + testing=True) + + dframe_explorer.geodataframe_explore(simple_geojson, testing=True) From ed4e5f42ce1bc817d8425f88fb21daa674101c18 Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Tue, 2 Sep 2014 15:55:44 -0700 Subject: [PATCH 06/11] geodataframe_explore -> gdf_explore (rename) --- urbansim/maps/dframe_explorer.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/urbansim/maps/dframe_explorer.py b/urbansim/maps/dframe_explorer.py index 8d5a6653..5f63cf18 100644 --- a/urbansim/maps/dframe_explorer.py +++ b/urbansim/maps/dframe_explorer.py @@ -158,26 +158,27 @@ def start(views, run(host=host, port=port, debug=True) -def geodataframe_explore(gdf, - dataframe_d=None, - center=None, - zoom=11, - geom_name=None, # from JSON file, use index if None - join_name='zone_id', # from data frames - precision=2, - port=8765, - host='localhost', - testing=False): +def gdf_explore(gdf, + dataframe_d=None, + center=None, + zoom=11, + geom_name=None, # from JSON file, use index if None + join_name='zone_id', # from data frames + precision=2, + port=8765, + host='localhost', + testing=False): """ - This method wraps the start method above. The parameters are the same as - above but many are optional and the defaults can be derived from the - dataframe in the following way. + This method wraps the start method above, but for displaying a geopandas + geodataframe. The parameters are the same as above but many are optional + and the defaults can be derived from the dataframe in the following way. If you don't pass a dataframe_d, only the fields directly on the geodataframe will be available. The center will be derived from the center of the dataframe's bounding box. The geom_name is optional and if it is not set or set to None, the index of the geodataframe will be used - for joining attributes to shapes. + for joining attributes to shapes. Obviously shape_json in the above + method is not used - the shapes on the geodataframe are used directly. """ try: import geopandas From 8c287b4a8f896f5acf08f2140ffdc4e02b1e59b9 Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Tue, 2 Sep 2014 15:58:04 -0700 Subject: [PATCH 07/11] file is kept in the right directory --- urbansim/maps/tests/test_explorer.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/urbansim/maps/tests/test_explorer.py b/urbansim/maps/tests/test_explorer.py index d1b47d5f..f2976df7 100644 --- a/urbansim/maps/tests/test_explorer.py +++ b/urbansim/maps/tests/test_explorer.py @@ -1,4 +1,5 @@ import pytest +import os import pandas as pd import geopandas as gpd @@ -15,7 +16,8 @@ def simple_map_input(): @pytest.fixture def simple_geojson(): - return gpd.read_file("test.geojson") + return gpd.read_file(os.path.join(os.path.dirname(__file__), + "test.geojson")) def test_explorer(simple_map_input): @@ -46,7 +48,6 @@ def test_explorer(simple_map_input): def test_geodf_explorer(simple_map_input, simple_geojson): d = {"dfname": simple_map_input} - dframe_explorer.geodataframe_explore(simple_geojson, dataframe_d=d, - testing=True) + dframe_explorer.gdf_explore(simple_geojson, dataframe_d=d, testing=True) - dframe_explorer.geodataframe_explore(simple_geojson, testing=True) + dframe_explorer.gdf_explore(simple_geojson, testing=True) From ca6001dcd1853e199a231f7fcc21f4c269d0068c Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Tue, 2 Sep 2014 15:59:48 -0700 Subject: [PATCH 08/11] on code review - there was a debug statement left in --- urbansim/maps/dframe_explorer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/urbansim/maps/dframe_explorer.py b/urbansim/maps/dframe_explorer.py index 5f63cf18..c3a8cd6c 100644 --- a/urbansim/maps/dframe_explorer.py +++ b/urbansim/maps/dframe_explorer.py @@ -120,7 +120,6 @@ def start(views, global DFRAMES, CONFIG, SHAPES DFRAMES = {str(k): views[k] for k in views} - print shape_json if not testing and not os.path.exists(shape_json): # if the file doesn't exist, we try to use it as json try: From 98f47144b309f1a6d81ae2bce9aac6ef911eb728 Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Tue, 2 Sep 2014 16:01:57 -0700 Subject: [PATCH 09/11] add geopandas to the pip install line --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b9459f56..ce1e0be7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ install: - | conda create -p $HOME/py --yes fiona ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pandana pytables pytest pyyaml shapely toolz "python=$TRAVIS_PYTHON_VERSION" -c "synthicity" - export PATH=$HOME/py/bin:$PATH -- pip install simplejson bottle +- pip install simplejson bottle geopandas - pip install pytest-cov coveralls pep8 - pip install . before_script: From 1f60910f2acb0cf90b22b96274a9c6626202fa42 Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Tue, 2 Sep 2014 16:11:29 -0700 Subject: [PATCH 10/11] I'm not sure why this test wasn't breaking locally. For now, I'm going to assume the shape being passed in is in epsg=4326. --- urbansim/maps/dframe_explorer.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/urbansim/maps/dframe_explorer.py b/urbansim/maps/dframe_explorer.py index c3a8cd6c..1cff69e8 100644 --- a/urbansim/maps/dframe_explorer.py +++ b/urbansim/maps/dframe_explorer.py @@ -172,6 +172,10 @@ def gdf_explore(gdf, geodataframe. The parameters are the same as above but many are optional and the defaults can be derived from the dataframe in the following way. + You are responsible for converting to crs 4326 - using the to_crs method + on the geodataframe (since we don't want to do this conversion every time + and geopandas doesn't check the current crs before converting). + If you don't pass a dataframe_d, only the fields directly on the geodataframe will be available. The center will be derived from the center of the dataframe's bounding box. The geom_name is optional and if @@ -195,20 +199,19 @@ def gdf_explore(gdf, dataframe_d["local"] = df # need to check if it's already 4326 - if gdf.crs != 4326: - self = gdf.to_crs(epsg=4326) + #gdf = gdf.to_crs(epsg=4326) - bbox = self.total_bounds + bbox = gdf.total_bounds if center is None: center = [(bbox[1]+bbox[3])/2, (bbox[0]+bbox[2])/2] - self.to_json() + gdf.to_json() start( dataframe_d, center=center, zoom=zoom, - shape_json=self.to_json(), + shape_json=gdf.to_json(), geom_name=geom_name, # from JSON file join_name=join_name, # from data frames precision=precision, From b6aa0fd3b0e0d808f29ad2f77165bed3bdafb0bb Mon Sep 17 00:00:00 2001 From: Fletcher Foti Date: Tue, 2 Sep 2014 16:16:05 -0700 Subject: [PATCH 11/11] pep8 --- urbansim/maps/dframe_explorer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbansim/maps/dframe_explorer.py b/urbansim/maps/dframe_explorer.py index 1cff69e8..8a352141 100644 --- a/urbansim/maps/dframe_explorer.py +++ b/urbansim/maps/dframe_explorer.py @@ -199,7 +199,7 @@ def gdf_explore(gdf, dataframe_d["local"] = df # need to check if it's already 4326 - #gdf = gdf.to_crs(epsg=4326) + # gdf = gdf.to_crs(epsg=4326) bbox = gdf.total_bounds if center is None: