Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Time-Space Diagrams automatically to S3 #993

Merged
merged 6 commits into from
Jul 8, 2020
Merged

Time-Space Diagrams automatically to S3 #993

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4361240
Add time-space diagram plotting to experiment.py
liljonnystyle Jul 8, 2020
731008a
Update experiment.py
liljonnystyle Jul 8, 2020
7bf1ede
fix pydocstyle and flake8 issues
liljonnystyle Jul 8, 2020
d01b8ea
Merge branch 'jl-tsd-to-s3' of https://github.com/flow-project/flow i…
liljonnystyle Jul 8, 2020
2711d1a
Merge branch 'i210_dev' of https://github.com/flow-project/flow into …
liljonnystyle Jul 8, 2020
8895ff8
only plot if to_aws flag is on
liljonnystyle Jul 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions flow/core/experiment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from flow.utils.registry import make_create_env
from flow.data_pipeline.data_pipeline import write_dict_to_csv, upload_to_s3, get_extra_info, get_configuration
from flow.data_pipeline.leaderboard_utils import network_name_translate
from flow.visualize.time_space_diagram import tsd_main
from collections import defaultdict
from datetime import datetime, timezone
import logging
Expand Down Expand Up @@ -231,6 +232,7 @@ def rl_actions(*_):

write_dict_to_csv(trajectory_table_path, extra_info)
write_dict_to_csv(metadata_table_path, metadata, True)
tsd_main(trajectory_table_path, flow_params, min_speed=0, max_speed=10, start=self.env.env_params.warmup_steps)

if to_aws:
upload_to_s3('circles.data.pipeline',
Expand All @@ -241,5 +243,8 @@ def rl_actions(*_):
'fact_vehicle_trace/date={0}/partition_name={1}/{1}.csv'.format(cur_date, source_id),
trajectory_table_path,
{'network': metadata['network'][0], 'is_baseline': metadata['is_baseline'][0]})
upload_to_s3('circles.data.pipeline',
'time_space_diagram/date={0}/partition_name={1}/{1}.csv'.format(cur_date, source_id),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the S3 key for the diagram should have .png at the end here

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah thanks!

trajectory_table_path.replace('csv', 'png'))

return info_dict
131 changes: 89 additions & 42 deletions flow/visualize/time_space_diagram.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ def _get_abs_pos(df, params):
return ret


def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None):
def plot_tsd(ax, df, segs, min_speed=0, max_speed=10, start=0, lane=None, ghost_edges=None, ghost_bounds=None):
"""Plot the time-space diagram.

Take the pre-processed segments and other meta-data, then plot all the line segments.
Expand All @@ -395,8 +395,12 @@ def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None)
data used for axes bounds and speed coloring
segs : list of list of lists
line segments to be plotted, where each segment is a list of two [x,y] pairs
args : dict
parsed arguments
min_speed : int or float
minimum speed in colorbar
max_speed : int or float
maximum speed in colorbar
start : int or float
starting time_step not greyed out
lane : int, optional
lane number to be shown in plot title
ghost_edges : list or set of str
Expand Down Expand Up @@ -457,41 +461,28 @@ def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None)
cbar.ax.tick_params(labelsize=18)


if __name__ == '__main__':
# create the parser
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description='[Flow] Generates time space diagrams for flow networks.',
epilog='python time_space_diagram.py </path/to/emission>.csv '
'</path/to/flow_params>.json')

# required arguments
parser.add_argument('trajectory_path', type=str,
help='path to the Flow trajectory csv file.')
parser.add_argument('flow_params', type=str,
help='path to the flow_params json file.')

# optional arguments
parser.add_argument('--steps', type=int, default=1,
help='rate at which steps are plotted.')
parser.add_argument('--title', type=str, default='Time Space Diagram',
help='rate at which steps are plotted.')
parser.add_argument('--max_speed', type=int, default=8,
help='The maximum speed in the color range.')
parser.add_argument('--min_speed', type=int, default=0,
help='The minimum speed in the color range.')
parser.add_argument('--start', type=float, default=0,
help='initial time (in sec) in the plot.')

args = parser.parse_args()

# flow_params is imported as a dictionary
if '.json' in args.flow_params:
flow_params = get_flow_params(args.flow_params)
else:
module = __import__("examples.exp_configs.non_rl", fromlist=[args.flow_params])
flow_params = getattr(module, args.flow_params).flow_params
def tsd_main(trajectory_path, flow_params, min_speed=0, max_speed=10, start=0):
"""Main function for plotting time-space diagram.

Parameters
----------
trajectory_path : str
file path (for the .csv formatted file)
flow_params : dict
flow-specific parameters, including:
* "network" (str): name of the network that was used when generating
the emission file. Must be one of the network names mentioned in
ACCEPTABLE_NETWORKS,
* "net_params" (flow.core.params.NetParams): network-specific
parameters. This is used to collect the lengths of various network
links.
min_speed : int or float
minimum speed in colorbar
max_speed : int or float
maximum speed in colorbar
start : int or float
starting time_step not greyed out
"""
# some plotting parameters
cdict = {
'red': ((0, 0, 0), (0.2, 1, 1), (0.6, 1, 1), (1, 0, 0)),
Expand All @@ -501,7 +492,7 @@ def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None)
my_cmap = colors.LinearSegmentedColormap('my_colormap', cdict, 1024)

# Read trajectory csv into pandas dataframe
traj_df = import_data_from_trajectory(args.trajectory_path, flow_params)
traj_df = import_data_from_trajectory(trajectory_path, flow_params)

# Convert df data into segments for plotting
segs, traj_df = get_time_space_data(traj_df, flow_params)
Expand All @@ -513,17 +504,35 @@ def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None)
for lane, df in traj_df.groupby('lane_id'):
ax = plt.subplot(nlanes, 1, lane+1)

plot_tsd(ax, df, segs[lane], args, int(lane+1), ghost_edges={'ghost0', '119257908#3'})
plot_tsd(ax=ax,
df=df,
segs=segs[lane],
min_speed=min_speed,
max_speed=max_speed,
start=start,
lane=int(lane+1),
ghost_edges={'ghost0', '119257908#3'})
plt.tight_layout()
else:
# perform plotting operation
fig = plt.figure(figsize=(16, 9))
ax = plt.axes()

if flow_params['network'] == HighwayNetwork:
plot_tsd(ax, traj_df, segs, args, ghost_bounds=(500, 2300))
plot_tsd(ax=ax,
df=traj_df,
segs=segs,
min_speed=min_speed,
max_speed=max_speed,
start=start,
ghost_bounds=(500, 2300))
else:
plot_tsd(ax, traj_df, segs, args)
plot_tsd(ax=ax,
df=traj_df,
segs=segs,
min_speed=min_speed,
max_speed=max_speed,
start=start)

###########################################################################
# Note: For MergeNetwork only #
Expand All @@ -534,5 +543,43 @@ def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None)
[-0.1, -0.1], linewidth=3, color="white") #
###########################################################################

outfile = args.trajectory_path.replace('csv', 'png')
outfile = trajectory_path.replace('csv', 'png')
plt.savefig(outfile)


if __name__ == '__main__':
# create the parser
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description='[Flow] Generates time space diagrams for flow networks.',
epilog='python time_space_diagram.py </path/to/emission>.csv '
'</path/to/flow_params>.json')

# required arguments
parser.add_argument('trajectory_path', type=str,
help='path to the Flow trajectory csv file.')
parser.add_argument('flow_params', type=str,
help='path to the flow_params json file.')

# optional arguments
parser.add_argument('--steps', type=int, default=1,
help='rate at which steps are plotted.')
parser.add_argument('--title', type=str, default='Time Space Diagram',
help='rate at which steps are plotted.')
parser.add_argument('--max_speed', type=int, default=8,
help='The maximum speed in the color range.')
parser.add_argument('--min_speed', type=int, default=0,
help='The minimum speed in the color range.')
parser.add_argument('--start', type=float, default=0,
help='initial time (in sec) in the plot.')

args = parser.parse_args()

# flow_params is imported as a dictionary
if '.json' in args.flow_params:
flow_params = get_flow_params(args.flow_params)
else:
module = __import__("examples.exp_configs.non_rl", fromlist=[args.flow_params])
flow_params = getattr(module, args.flow_params).flow_params

tsd_main(args.trajectory_path, flow_params, min_speed=args.min_speed, max_speed=args.max_speed, start=args.start)