From d695b2378a0d8dd5dda1fcb7bbcda73affaf2d13 Mon Sep 17 00:00:00 2001 From: msoltadeo Date: Tue, 17 Mar 2020 11:13:53 -0300 Subject: [PATCH 01/12] updates for new pandas version --- urbansim/models/dcm.py | 8 ++++---- urbansim/urbanchoice/tests/test_mnl.py | 14 +++++++------- urbansim/utils/misc.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/urbansim/models/dcm.py b/urbansim/models/dcm.py index b649ad1d..fc62eeeb 100644 --- a/urbansim/models/dcm.py +++ b/urbansim/models/dcm.py @@ -413,14 +413,14 @@ def fit(self, choosers, alternatives, current_choice): model_design = dmatrix( self.str_model_expression, data=merged, return_type='dataframe') - if len(merged) != model_design.as_matrix().shape[0]: + if len(merged) != model_design.values.shape[0]: raise ModelEvaluationError( 'Estimated data does not have the same length as input. ' 'This suggests there are null values in one or more of ' 'the input columns.') self.log_likelihoods, self.fit_parameters = mnl.mnl_estimate( - model_design.as_matrix(), chosen, self.sample_size) + model_design.values, chosen, self.sample_size) self.fit_parameters.index = model_design.columns logger.debug('finish: fit LCM model {}'.format(self.name)) @@ -523,7 +523,7 @@ def probabilities(self, choosers, alternatives, filter_tables=True): model_design = dmatrix( self.str_model_expression, data=merged, return_type='dataframe') - if len(merged) != model_design.as_matrix().shape[0]: + if len(merged) != model_design.values.shape[0]: raise ModelEvaluationError( 'Simulated data does not have the same length as input. ' 'This suggests there are null values in one or more of ' @@ -542,7 +542,7 @@ def probabilities(self, choosers, alternatives, filter_tables=True): numalts = sample_size probabilities = mnl.mnl_simulate( - model_design.as_matrix(), + model_design.values, coeffs, numalts=numalts, returnprobs=True) diff --git a/urbansim/urbanchoice/tests/test_mnl.py b/urbansim/urbanchoice/tests/test_mnl.py index a939e462..d9a067d5 100644 --- a/urbansim/urbanchoice/tests/test_mnl.py +++ b/urbansim/urbanchoice/tests/test_mnl.py @@ -110,12 +110,12 @@ def choosers_dm(choosers, test_data): @pytest.fixture def fit_coeffs(dm, chosen, num_alts): - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values), chosen, num_alts) return fit.Coefficient.values def test_mnl_estimate(dm, chosen, num_alts, test_data): - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values), chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(test_data['est_expected']) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -134,10 +134,10 @@ def test_mnl_simulate(dm, fit_coeffs, num_alts, test_data, choosers_dm): # now test with real data probs = mnl.mnl_simulate( - choosers_dm.as_matrix(), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values), fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=test_data['sim_expected'].columns) results, expected = results.align(test_data['sim_expected']) - npt.assert_allclose(results.as_matrix(), expected.as_matrix(), rtol=1e-4) + npt.assert_allclose(results.values), expected.values), rtol=1e-4) def test_alternative_specific_coeffs(num_alts): @@ -193,7 +193,7 @@ def test_alternative_specific_coeffs(num_alts): 'boat:(intercept)', 'charter:(intercept)', 'pier:(intercept)', 'boat:income', 'charter:income', 'pier:income']) - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), fish_chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values), fish_chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(expected) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -206,7 +206,7 @@ def test_alternative_specific_coeffs(num_alts): fit_coeffs = fit.Coefficient.values probs = mnl.mnl_simulate( - choosers_dm.as_matrix(), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values), fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=expected.columns) results, expected = results.align(expected) - npt.assert_allclose(results.as_matrix(), expected.as_matrix(), rtol=1e-4) + npt.assert_allclose(results.values), expected.values), rtol=1e-4) diff --git a/urbansim/utils/misc.py b/urbansim/utils/misc.py index b2aadfd3..3d7b8350 100644 --- a/urbansim/utils/misc.py +++ b/urbansim/utils/misc.py @@ -138,7 +138,7 @@ def compute_range(travel_data, attr, travel_time_attr, dist, agg=np.sum): """ travel_data = travel_data.reset_index(level=1) travel_data = travel_data[travel_data[travel_time_attr] < dist] - travel_data["attr"] = attr[travel_data.to_zone_id].values + travel_data["attr"] = attr.reindex(travel_data.to_zone_id, fill_value=0).values return travel_data.groupby(level=0).attr.apply(agg) From 9c43d04702f1c2580d075ebb5619eb31a3700d68 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 11:03:59 -0700 Subject: [PATCH 02/12] Fixing typos in search-and-replace --- urbansim/urbanchoice/tests/test_mnl.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/urbansim/urbanchoice/tests/test_mnl.py b/urbansim/urbanchoice/tests/test_mnl.py index d9a067d5..ff6ef4cf 100644 --- a/urbansim/urbanchoice/tests/test_mnl.py +++ b/urbansim/urbanchoice/tests/test_mnl.py @@ -110,12 +110,12 @@ def choosers_dm(choosers, test_data): @pytest.fixture def fit_coeffs(dm, chosen, num_alts): - log_like, fit = mnl.mnl_estimate(dm.values), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, chosen, num_alts) return fit.Coefficient.values def test_mnl_estimate(dm, chosen, num_alts, test_data): - log_like, fit = mnl.mnl_estimate(dm.values), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(test_data['est_expected']) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -134,10 +134,10 @@ def test_mnl_simulate(dm, fit_coeffs, num_alts, test_data, choosers_dm): # now test with real data probs = mnl.mnl_simulate( - choosers_dm.values), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values, fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=test_data['sim_expected'].columns) results, expected = results.align(test_data['sim_expected']) - npt.assert_allclose(results.values), expected.values), rtol=1e-4) + npt.assert_allclose(results.values, expected.values, rtol=1e-4) def test_alternative_specific_coeffs(num_alts): @@ -193,7 +193,7 @@ def test_alternative_specific_coeffs(num_alts): 'boat:(intercept)', 'charter:(intercept)', 'pier:(intercept)', 'boat:income', 'charter:income', 'pier:income']) - log_like, fit = mnl.mnl_estimate(dm.values), fish_chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, fish_chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(expected) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -206,7 +206,7 @@ def test_alternative_specific_coeffs(num_alts): fit_coeffs = fit.Coefficient.values probs = mnl.mnl_simulate( - choosers_dm.values), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values, fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=expected.columns) results, expected = results.align(expected) - npt.assert_allclose(results.values), expected.values), rtol=1e-4) + npt.assert_allclose(results.values, expected.values, rtol=1e-4) From 61b1d50e74deff96d8cdf83c7b20ad564735190f Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 14:33:35 -0700 Subject: [PATCH 03/12] Fixing column list comparison --- urbansim/urbanchoice/tests/test_interaction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urbansim/urbanchoice/tests/test_interaction.py b/urbansim/urbanchoice/tests/test_interaction.py index 938edcf4..aeca68a5 100644 --- a/urbansim/urbanchoice/tests/test_interaction.py +++ b/urbansim/urbanchoice/tests/test_interaction.py @@ -35,8 +35,8 @@ def test_interaction_dataset_sim(choosers, alternatives): assert len(merged) == len(choosers) * len(alternatives) npt.assert_array_equal(merged.index.values, sample) - assert list(merged.columns) == [ - 'var2', 'var3', 'join_index', 'thing_id', 'var1'] + assert set(list(merged.columns)) == set([ + 'var2', 'var3', 'join_index', 'thing_id', 'var1']) npt.assert_array_equal( merged['var1'].values, choosers['var1'].values.repeat(len(alternatives))) From 2954eb2e11a447caa1eeda4d21139e398ec81364 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 14:38:24 -0700 Subject: [PATCH 04/12] .ix[] -> .loc[] --- urbansim/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbansim/utils/misc.py b/urbansim/utils/misc.py index 3d7b8350..e758a560 100644 --- a/urbansim/utils/misc.py +++ b/urbansim/utils/misc.py @@ -357,7 +357,7 @@ def series64bitto32bit(s): def _pandassummarytojson(v, ndigits=3): - return {i: round(float(v.ix[i]), ndigits) for i in v.index} + return {i: round(float(v.loc[i]), ndigits) for i in v.index} def pandasdfsummarytojson(df, ndigits=3): From 893732e5b5cce86f54f6897e75dcc6fdc13924f9 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:06:06 -0700 Subject: [PATCH 05/12] Updating travis environment --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 15b74bbe..55d656c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,8 @@ language: python sudo: false python: - '2.7' -- '3.5' +- '3.6' +- '3.8' install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh @@ -16,11 +17,11 @@ install: - > conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - cytoolz ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy - statsmodels pytables pytest pyyaml toolz + cytoolz jupyter jinja2 matplotlib numpy pandas pandana patsy pip scipy + statsmodels pytables pytest pyyaml toolz prettytable - source activate test-environment - pip install orca osmnet pandana bottle simplejson zbox -- pip install pytest-cov coveralls pycodestyle +- pip install 'pytest<4.0' pytest-cov coveralls pycodestyle - pip install . before_script: - git clone https://github.com/udst/sanfran_urbansim.git From f698273c6f70a438700400e650e1ccb5471f26d5 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:17:26 -0700 Subject: [PATCH 06/12] Travis environment adjustments --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55d656c4..0486f6fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,11 @@ install: - > conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - cytoolz jupyter jinja2 matplotlib numpy pandas pandana patsy pip scipy - statsmodels pytables pytest pyyaml toolz prettytable + cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy + statsmodels pytables pyyaml toolz - source activate test-environment -- pip install orca osmnet pandana bottle simplejson zbox +- conda install pandana -c conda-forge +- pip install orca osmnet pandana bottle simplejson zbox prettytable - pip install 'pytest<4.0' pytest-cov coveralls pycodestyle - pip install . before_script: From 016ff41c2a7930062ef1d1b595f8f777765a3005 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:25:09 -0700 Subject: [PATCH 07/12] Updating appveyor environment --- appveyor.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 075cdeca..b1dbde08 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,7 @@ environment: matrix: - PYTHON_VERSION: 2.7 MINICONDA: C:\Miniconda - - PYTHON_VERSION: 3.5 + - PYTHON_VERSION: 3.6 MINICONDA: C:\Miniconda3 init: @@ -15,11 +15,11 @@ install: - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda info -a - - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pytest pyyaml toolz" + - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pyyaml toolz" - activate test-environment - - conda install -c conda-forge shapely geopandas - - pip install orca osmnet pandana bottle simplejson zbox - - pip install pycodestyle + - conda install -c conda-forge shapely geopandas pandana + - pip install orca osmnet bottle simplejson zbox prettytable + - pip install 'pytest<4.0' pycodestyle - pip install . before_test: From 3200e8acc5ceb915abbe4b6e9798de3f48975880 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:37:34 -0700 Subject: [PATCH 08/12] Switching pandana from conda to pip for appveyor --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b1dbde08..e7776993 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,8 +17,8 @@ install: - conda info -a - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pyyaml toolz" - activate test-environment - - conda install -c conda-forge shapely geopandas pandana - - pip install orca osmnet bottle simplejson zbox prettytable + - conda install -c conda-forge shapely geopandas + - pip install orca pandana osmnet bottle simplejson zbox prettytable - pip install 'pytest<4.0' pycodestyle - pip install . From 6c2cbb044a6eec9731bc59a860de842436c42138 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:59:23 -0700 Subject: [PATCH 09/12] AppVeyor syntax --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index e7776993..df40f72a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,7 +19,7 @@ install: - activate test-environment - conda install -c conda-forge shapely geopandas - pip install orca pandana osmnet bottle simplejson zbox prettytable - - pip install 'pytest<4.0' pycodestyle + - pip install pytest<4.0 pycodestyle - pip install . before_test: From a437a81f2328bd692289615fba1ead3f28e97166 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 17:53:10 -0700 Subject: [PATCH 10/12] Another try --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index df40f72a..e9d0bb7e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,7 +19,7 @@ install: - activate test-environment - conda install -c conda-forge shapely geopandas - pip install orca pandana osmnet bottle simplejson zbox prettytable - - pip install pytest<4.0 pycodestyle + - "pip install pytest<4.0 pycodestyle" - pip install . before_test: From effd3a8df22c91731ac46c6f6cb7ed3357cc7877 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 19 Mar 2020 21:12:40 -0700 Subject: [PATCH 11/12] Pytest pip -> conda for appveyor --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e9d0bb7e..0ab7faf2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,9 +17,9 @@ install: - conda info -a - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pyyaml toolz" - activate test-environment - - conda install -c conda-forge shapely geopandas + - conda install -c conda-forge shapely geopandas pytest=3.10 - pip install orca pandana osmnet bottle simplejson zbox prettytable - - "pip install pytest<4.0 pycodestyle" + - pip install pycodestyle - pip install . before_test: From c9b49115253bfe7f28d53a8de263cc9e6bdcff18 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 23 Mar 2020 14:24:40 -0700 Subject: [PATCH 12/12] Revising tests that now fail in win py27 --- urbansim/utils/tests/test_testing.py | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/urbansim/utils/tests/test_testing.py b/urbansim/utils/tests/test_testing.py index 642e8a56..f2ad800f 100644 --- a/urbansim/utils/tests/test_testing.py +++ b/urbansim/utils/tests/test_testing.py @@ -17,36 +17,36 @@ def test_frames_equal_mismatched_columns(): expected = pd.DataFrame({'a': [1]}) actual = pd.DataFrame({'b': [2]}) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == "Expected column 'a' not found." + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal_mismatched_rows(): expected = pd.DataFrame({'a': [1]}, index=[0]) actual = pd.DataFrame({'a': [1]}, index=[1]) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == "Expected row 0 not found." + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal_mismatched_items(): expected = pd.DataFrame({'a': [1]}) actual = pd.DataFrame({'a': [2]}) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == """ -Items are not equal: - ACTUAL: 2 - DESIRED: 1 - -Column: 'a' -Row: 0""" + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal():