Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add profile plotter and pyswmm integration #165

Merged
merged 24 commits into from
Dec 29, 2022
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
c199624
updated requirements.txt - added geopandas
bemcdonnell Sep 27, 2022
725330b
First steps to Migrate to PySWMM as dependency
bemcdonnell Sep 28, 2022
c05fbfd
Added Network Tracer function and tests
bemcdonnell Dec 22, 2022
d9127c3
Merge branch 'master' of https://github.com/aerispaha/swmmio
bemcdonnell Dec 22, 2022
0076c19
fix merge
bemcdonnell Dec 22, 2022
58f5e3c
removed line
bemcdonnell Dec 22, 2022
5024faf
Adding profile plotting code
bemcdonnell Dec 23, 2022
13529da
added function docstrings
bemcdonnell Dec 23, 2022
e5961c1
added import error
bemcdonnell Dec 23, 2022
78e5a68
Adding unit test for profiler config info
bemcdonnell Dec 23, 2022
1a705cd
updated requiredments
bemcdonnell Dec 23, 2022
7d17665
add unit tests for and refactor run_models.run
aerispaha Dec 28, 2022
f332b05
add report to inp, use in run.py
aerispaha Dec 28, 2022
b111808
add simple test of swmmio cli
aerispaha Dec 29, 2022
3f73c42
use jerzy example model for cli test
aerispaha Dec 29, 2022
350aae9
remove unnecessary files from run_models module
aerispaha Dec 29, 2022
e8d9849
step commit before pulling changes
bemcdonnell Dec 29, 2022
148fc89
Merge branch 'profile_plotter' of https://github.com/aerispaha/swmmio…
bemcdonnell Dec 29, 2022
3c2c768
fix
bemcdonnell Dec 29, 2022
ce5215f
syntax tweaks in utils.functions
aerispaha Dec 29, 2022
1748d2c
add temp unit test duplicate using temp directory
aerispaha Dec 29, 2022
c739da8
syntax tweaks in test_functions
aerispaha Dec 29, 2022
0d0735d
rm Example1_parallel_loop.out and Example1_parallel_loop.rpt
aerispaha Dec 29, 2022
83a598d
refactor test_profile such that it doesn't leave .out and .rpt artifacts
aerispaha Dec 29, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file removed lib/linux/swmm5
Binary file not shown.
Binary file removed lib/windows/swmm5_22.exe
Binary file not shown.
3 changes: 3 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,6 @@ m2r
mistune==0.8.4
# Run dependencies
pyproj>=3.0.0
geopandas
matplotlib
pyswmm>=1.2
3 changes: 3 additions & 0 deletions swmmio/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
from swmmio.elements import *
from swmmio.version_control import *
from swmmio.utils.dataframes import dataframe_from_bi, dataframe_from_rpt, dataframe_from_inp
from swmmio.utils.functions import find_network_trace
from swmmio.graphics.swmm_graphics import create_map
from swmmio.graphics.profiler import (build_profile_plot, add_hgl_plot,
add_node_labels_plot, add_link_labels_plot)

# import swmmio.core as swmmio
'''Python SWMM Input/Output Tools'''
Expand Down
87 changes: 45 additions & 42 deletions swmmio/__main__.py
Original file line number Diff line number Diff line change
@@ -1,58 +1,61 @@
import argparse
import os
from itertools import chain

from swmmio.run_models.run import run_simple, run_hot_start_sequence
from swmmio.run_models import start_pool

from swmmio import Model
from itertools import chain
import os
import argparse
from multiprocessing import Pool, cpu_count
from datetime import datetime

#parse the arguments
parser = argparse.ArgumentParser(description='Process some stuff')
parser.add_argument('-r', '--run', dest='model_to_run', nargs="+")
parser.add_argument('-rhs', '--run_hotstart', dest='hotstart_model_to_run', nargs="+")
parser.add_argument('-sp', '--start_pool', dest='start_pool', nargs="+")
parser.add_argument('-cores_left', '--cores_left', dest='cores_left', default=4, type=int)
parser.add_argument('-pp', '--post_process', dest='post_process', nargs="+")
def main():
# parse the arguments
parser = argparse.ArgumentParser(description='Process some stuff')
parser.add_argument('-r', '--run', dest='model_to_run', nargs="+")
parser.add_argument('-rhs', '--run_hotstart', dest='hotstart_model_to_run', nargs="+")
parser.add_argument('-sp', '--start_pool', dest='start_pool', nargs="+")
parser.add_argument('-cores_left', '--cores_left', dest='cores_left', default=4, type=int)
parser.add_argument('-pp', '--post_process', dest='post_process', nargs="+")

args = parser.parse_args()
wd = os.getcwd() # current directory script is being called from

args = parser.parse_args()
wd = os.getcwd() #current directory script is being called from
if args.model_to_run is not None:

if args.model_to_run is not None:
models_paths = [os.path.join(wd, f) for f in args.model_to_run]
print('Adding models to queue:\n\t{}'.format('\n\t'.join(models_paths)))

models_paths = [os.path.join(wd, f) for f in args.model_to_run]
print('Adding models to queue:\n\t{}'.format('\n\t'.join(models_paths)))
# run the models in series (one after the other)
list(map(run_simple, models_paths))
# run_simple(args.model_to_run)

#run the models in series (one after the other)
list(map(run_simple, models_paths))
# run_simple(args.model_to_run)
elif args.hotstart_model_to_run is not None:
models_paths = [os.path.join(wd, f) for f in args.hotstart_model_to_run]
print('hotstart_model_to_run the model: {}'.format(args.hotstart_model_to_run))
# m = Model(args.hotstart_model_to_run)
# run_hot_start_sequence(m)#args.hotstart_model_to_run)
list(map(run_hot_start_sequence, models_paths))

elif args.hotstart_model_to_run is not None:
models_paths = [os.path.join(wd, f) for f in args.hotstart_model_to_run]
print('hotstart_model_to_run the model: {}'.format(args.hotstart_model_to_run))
# m = Model(args.hotstart_model_to_run)
# run_hot_start_sequence(m)#args.hotstart_model_to_run)
list(map(run_hot_start_sequence, models_paths))
elif args.start_pool is not None:

elif args.start_pool is not None:
models_dirs = [os.path.join(wd, f) for f in args.start_pool]
print('Searching for models in:\n\t{}'.format('\n\t'.join(models_dirs)))
# combine the segments and options (combinations) into one iterable
inp_paths = []
for root, dirs, files in chain.from_iterable(os.walk(path) for path in models_dirs):
for f in files:
if f.endswith('.inp') and 'bk' not in root:
# we've found a directory containing an inp
inp_paths.append(os.path.join(root, f))

models_dirs = [os.path.join(wd, f) for f in args.start_pool]
print('Searching for models in:\n\t{}'.format('\n\t'.join(models_dirs)))
#combine the segments and options (combinations) into one iterable
inp_paths = []
for root, dirs, files in chain.from_iterable(os.walk(path) for path in models_dirs):
for f in files:
if f.endswith('.inp') and 'bk' not in root:
#we've found a directory containing an inp
inp_paths.append(os.path.join(root, f))
# call the main() function in start_pool.py
start_pool.main(inp_paths, args.cores_left)

print("swmmio has completed running {} models".format(len(inp_paths)))

#call the main() function in start_pool.py
start_pool.main(inp_paths, args.cores_left)
else:
print('you need to pass in some args')

print("swmmio has completed running {} models".format(len(inp_paths)))
return 0


else:
print('you need to pass in some args')
if __name__ == '__main__':
main()
38 changes: 34 additions & 4 deletions swmmio/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -504,6 +504,7 @@ class inp(SWMMIOFile):
def __init__(self, file_path):
self._options_df = None
self._files_df = None
self._report_df = None
self._conduits_df = None
self._xsections_df = None
self._pumps_df = None
Expand All @@ -529,6 +530,7 @@ def __init__(self, file_path):
self._sections = [
'[OPTIONS]',
'[FILES]',
'[REPORT]',
'[CONDUITS]',
'[XSECTIONS]',
'[PUMPS]',
Expand Down Expand Up @@ -562,9 +564,12 @@ def save(self, target_path=None):
"""
from swmmio.utils.modify_model import replace_inp_section
import shutil
target_path = target_path if target_path is not None else self.path

shutil.copyfile(self.path, target_path)
if target_path is not None:
shutil.copyfile(self.path, target_path)
else:
target_path = self.path

for section in self._sections:
# reformate the [SECTION] to section (and _section_df)
sect_id = section.translate({ord(i): None for i in '[]'}).lower()
Expand Down Expand Up @@ -651,8 +656,6 @@ def files(self):

:return: files section of the INP file
:rtype: pandas.DataFrame

Examples:
"""
if self._files_df is None:
self._files_df = dataframe_from_inp(self.path, "[FILES]")
Expand All @@ -664,6 +667,33 @@ def files(self, df):
first_col = df.columns[0]
self._files_df = df.set_index(first_col)

@property
def report(self):
"""
Get/set report section of the INP file.

:return: report section of the INP file
:rtype: pandas.DataFrame

>>> from swmmio.examples import jersey
>>> jersey.inp.report #doctest: +NORMALIZE_WHITESPACE
Status
Param
INPUT YES
CONTROLS YES
SUBCATCHMENTS NONE
NODES ALL
LINKS NONE
"""
if self._report_df is None:
self._report_df = dataframe_from_inp(self.path, "report")
return self._report_df

@report.setter
def report(self, df):
"""Set inp.report DataFrame."""
self._report_df = df

@property
def conduits(self):
"""
Expand Down
9 changes: 4 additions & 5 deletions swmmio/defs/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,8 @@
# This is the swmmio project root
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# path to the SWMM5 executable used within the run_models module
if os.name == 'posix':
SWMM_ENGINE_PATH = os.path.join(ROOT_DIR, 'lib', 'linux', 'swmm5')
else:
SWMM_ENGINE_PATH = os.path.join(ROOT_DIR, 'lib', 'windows', 'swmm5_22.exe')
# path to the Python executable used to run your version of Python
PYTHON_EXE_PATH = "python"#os.path.join(os.__file__.split("lib/")[0],"bin","python")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aerispaha can you give me some advice here? This could break someone's run if they had python pointing to py2.7

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, well we officially dropped support for Python 2.x back in version 0.3.7. Do you think we need to support 2.x to have pyswmm integrated as the model run engine?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aerispaha nope! If the user is not running inside an environment, this line just calls “python” rather than the version of python you are using to execute.


# feature class name of parcels in geodatabase
PARCEL_FEATURES = r'PWD_PARCELS_SHEDS_PPORT'
Expand All @@ -25,3 +22,5 @@
BASEMAP_PATH = os.path.join(ROOT_DIR, 'swmmio', 'reporting', 'basemaps', 'index.html')
BETTER_BASEMAP_PATH = os.path.join(ROOT_DIR, 'swmmio', 'reporting', 'basemaps', 'mapbox_base.html')

# PySWMM Wrapper Path
PYSWMM_WRAPPER_PATH = os.path.join(ROOT_DIR, 'swmmio', 'wrapper', 'pyswmm_wrapper.py')
Loading