From b93e6fe4cfa0fc7b3912895b50f9a0719ac6db92 Mon Sep 17 00:00:00 2001 From: aletzdy <58451076+aletzdy@users.noreply.github.com> Date: Fri, 30 Sep 2022 11:47:01 -0700 Subject: [PATCH 01/27] updated scripts to include simulation-based shadow pricing --- activitysim/abm/models/location_choice.py | 92 +++- activitysim/abm/tables/shadow_pricing.py | 581 ++++++++++++++++++++-- activitysim/core/mp_tasks.py | 34 ++ 3 files changed, 649 insertions(+), 58 deletions(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index f35d3eb15..330e669d4 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -227,7 +227,7 @@ def location_sample( DEST_MAZ = "dest_MAZ" -def aggregate_size_terms(dest_size_terms, network_los): +def aggregate_size_terms(dest_size_terms, network_los, model_settings): # # aggregate MAZ_size_terms to TAZ_size_terms # @@ -261,6 +261,19 @@ def aggregate_size_terms(dest_size_terms, network_los): for c in weighted_average_cols: TAZ_size_terms[c] /= TAZ_size_terms["size_term"] # weighted average + spc = shadow_pricing.load_shadow_price_calculator(model_settings) + if spc.shadow_settings["SHADOW_PRICE_METHOD"] == "simulation": + # allow TAZs with at least one underassigned MAZ in them, therefore with a shadowprice larger than -999, to be selected again + TAZ_size_terms["shadow_price_utility_adjustment"] = np.where( + TAZ_size_terms["shadow_price_utility_adjustment"] > -999, 0, -999 + ) + # now, negative size term means shadow price is -999. Setting size_term to 0 so the prob of that MAZ being selected becomes 0 + MAZ_size_terms["size_term"] = np.where( + MAZ_size_terms["shadow_price_utility_adjustment"] < 0, + 0, + MAZ_size_terms["size_term"], + ) + if TAZ_size_terms.isna().any(axis=None): logger.warning( f"TAZ_size_terms with NAN values\n{TAZ_size_terms[TAZ_size_terms.isna().any(axis=1)]}" @@ -308,7 +321,9 @@ def location_presample( alt_dest_col_name = model_settings["ALT_DEST_COL_NAME"] assert DEST_TAZ != alt_dest_col_name - MAZ_size_terms, TAZ_size_terms = aggregate_size_terms(dest_size_terms, network_los) + MAZ_size_terms, TAZ_size_terms = aggregate_size_terms( + dest_size_terms, network_los, model_settings + ) # convert MAZ zone_id to 'TAZ' in choosers (persons_merged) # persons_merged[HOME_TAZ] = persons_merged[HOME_MAZ].map(maz_to_taz) @@ -856,6 +871,7 @@ def iterate_location_choice( # chooser segmentation allows different sets coefficients for e.g. different income_segments or tour_types chooser_segment_column = model_settings["CHOOSER_SEGMENT_COLUMN_NAME"] + segment_ids = model_settings["SEGMENT_IDS"] assert ( chooser_segment_column in persons_merged_df @@ -869,11 +885,42 @@ def iterate_location_choice( for iteration in range(1, max_iterations + 1): + persons_merged_df_ = persons_merged_df.copy() + # print("################################_" + str(iteration)) + if spc.use_shadow_pricing and iteration > 1: spc.update_shadow_prices() - choices_df, save_sample_df = run_location_choice( - persons_merged_df, + if spc.shadow_settings["SHADOW_PRICE_METHOD"] == "simulation": + # filter from the sampled persons + persons_merged_df_ = persons_merged_df_[ + persons_merged_df_.index.isin(spc.sampled_persons.index) + ] + # handle cases where a segment has persons but no zones to receive them + desired_size_sum = spc.desired_size[ + spc.desired_size.index.isin( + spc.shadow_prices[spc.shadow_prices.iloc[:, 0] != -999].index + ) + ].sum() + zero_desired_size_segments = [ + i for i in desired_size_sum.index if desired_size_sum[i] == 0 + ] + zero_desired_size_segments_ids = [ + segment_ids[key] for key in zero_desired_size_segments + ] + persons_merged_df_ = persons_merged_df_[ + ~persons_merged_df_[chooser_segment_column].isin( + zero_desired_size_segments_ids + ) + ] + + persons_merged_df_ = persons_merged_df_.sort_index() + + # sample_df["n_" + str(iteration)] = spc.sampled_persons + # sample_df.to_csv(r"E:\Projects\Clients\SEMCOG\semcog_2zone_rundir\temp\sample_df" + ".csv") + + choices_df_, save_sample_df = run_location_choice( + persons_merged_df_, network_los, shadow_price_calculator=spc, want_logsums=logsum_column_name is not None, @@ -886,10 +933,35 @@ def iterate_location_choice( trace_label=tracing.extend_trace_label(trace_label, "i%s" % iteration), ) - # choices_df is a pandas DataFrame with columns 'choice' and (optionally) 'logsum' - if choices_df is None: + # choices_df is a pandas DataFrame with columns "choice" and (optionally) "logsum" + if choices_df_ is None: break + if spc.use_shadow_pricing: + # handle simulation method + if ( + spc.shadow_settings["SHADOW_PRICE_METHOD"] == "simulation" + and iteration > 1 + ): + # if a process ends up with no sampled workers in it, hence an empty choice_df_, then choice_df wil be what it was previously + if len(choices_df_) == 0: + choices_df = choices_df + else: + choices_df = pd.concat([choices_df, choices_df_], axis=0) + choices_df_index = choices_df_.index.name + choices_df = choices_df.reset_index() + # update choices of workers/students + choices_df = choices_df.drop_duplicates( + subset=[choices_df_index], keep="last" + ) + choices_df = choices_df.set_index(choices_df_index) + choices_df = choices_df.sort_index() + else: + choices_df = choices_df_.copy() + + else: + choices_df = choices_df_ + spc.set_choices( choices=choices_df["choice"], segment_ids=persons_merged_df[chooser_segment_column].reindex( @@ -901,13 +973,7 @@ def iterate_location_choice( spc.write_trace_files(iteration) if spc.use_shadow_pricing and spc.check_fit(iteration): - logging.info( - "%s converged after iteration %s" - % ( - trace_label, - iteration, - ) - ) + logging.info("%s converged after iteration %s" % (trace_label, iteration,)) break # - shadow price table diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 053b1dcbc..61a4e29a1 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -72,7 +72,14 @@ def size_table_name(model_selector): class ShadowPriceCalculator(object): def __init__( - self, model_settings, num_processes, shared_data=None, shared_data_lock=None + self, + model_settings, + num_processes, + shared_data=None, + shared_data_lock=None, + shared_data_choice=None, + shared_data_choice_lock=None, + shared_sp_choice_df=None, ): """ @@ -147,11 +154,19 @@ def __init__( self.shared_data = shared_data self.shared_data_lock = shared_data_lock + self.shared_data_choice = shared_data_choice + self.shared_data_choice_lock = shared_data_choice_lock + + self.shared_sp_choice_df = shared_sp_choice_df + self.shared_sp_choice_df = self.shared_sp_choice_df.astype("int") + self.shared_sp_choice_df = self.shared_sp_choice_df.set_index("person_id") + self.shared_sp_choice_df["choice"] = int(0) + # - load saved shadow_prices (if available) and set max_iterations accordingly if self.use_shadow_pricing: self.shadow_prices = None self.shadow_price_method = self.shadow_settings["SHADOW_PRICE_METHOD"] - assert self.shadow_price_method in ["daysim", "ctramp"] + assert self.shadow_price_method in ["daysim", "ctramp", "simulation"] if self.shadow_settings["LOAD_SAVED_SHADOW_PRICES"]: # read_saved_shadow_prices logs error and returns None if file not found @@ -182,6 +197,24 @@ def __init__( self.max_abs_diff = pd.DataFrame(index=self.desired_size.columns) self.max_rel_diff = pd.DataFrame(index=self.desired_size.columns) + if ( + self.use_shadow_pricing + and self.shadow_settings["SHADOW_PRICE_METHOD"] == "simulation" + ): + + assert self.model_selector in ["workplace", "school"] + + if self.model_selector == "workplace": + self.total_emp = self.shadow_settings["TOTAL_EMP"] + land_use = inject.get_table("land_use").to_frame() + self.target = land_use[self.total_emp] + + elif self.model_selector == "school": + total_enr = self.shadow_settings["TOTAL_ENORLLMENT"] + land_use = inject.get_table("land_use").to_frame() + self.target = land_use[total_enr] + self.zonal_sample_rate = None + def read_saved_shadow_prices(self, model_settings): """ Read saved shadow_prices from csv file in data_dir (so-called warm start) @@ -216,35 +249,25 @@ def read_saved_shadow_prices(self, model_settings): return shadow_prices - def synchronize_choices(self, local_modeled_size): + def synchronize_modeled_size(self, local_modeled_size): """ We have to wait until all processes have computed choices and aggregated them by segment and zone before we can compute global aggregate zone counts (by segment). Since the global zone counts are in shared data, we have to coordinate access to the data structure across sub-processes. - Note that all access to self.shared_data has to be protected by acquiring shared_data_lock - ShadowPriceCalculator.synchronize_choices coordinates access to the global aggregate zone counts (local_modeled_size summed across all sub-processes). - * All processes wait (in case we are iterating) until any stragglers from the previous iteration have exited the building. (TALLY_CHECKOUT goes to zero) - * Processes then add their local counts into the shared_data and increment TALLY_CHECKIN - * All processes wait until everybody has checked in (TALLY_CHECKIN == num_processes) - * Processes make local copy of shared_data and check out (increment TALLY_CHECKOUT) - * first_in process waits until all processes have checked out, then zeros shared_data and clears semaphores - Parameters ---------- local_modeled_size : pandas DataFrame - - Returns ------- global_modeled_size_df : pandas DataFrame @@ -303,6 +326,95 @@ def wait(tally, target): return global_modeled_size_df + def synchronize_choices(self, local_modeled_size): + """ + We have to wait until all processes have computed choices and aggregated them by segment + and zone before we can compute global aggregate zone counts (by segment). Since the global + zone counts are in shared data, we have to coordinate access to the data structure across + sub-processes. + + Note that all access to self.shared_data has to be protected by acquiring shared_data_lock + + ShadowPriceCalculator.synchronize_choices coordinates access to the global aggregate + zone counts (local_modeled_size summed across all sub-processes). + + * All processes wait (in case we are iterating) until any stragglers from the previous + iteration have exited the building. (TALLY_CHECKOUT goes to zero) + + * Processes then add their local counts into the shared_data and increment TALLY_CHECKIN + + * All processes wait until everybody has checked in (TALLY_CHECKIN == num_processes) + + * Processes make local copy of shared_data and check out (increment TALLY_CHECKOUT) + + * first_in process waits until all processes have checked out, then zeros shared_data + and clears semaphores + + Parameters + ---------- + local_modeled_size : pandas DataFrame + + + Returns + ------- + global_modeled_size_df : pandas DataFrame + local copy of shared global_modeled_size data as dataframe + with same shape and columns as local_modeled_size + """ + + # shouldn't be called if we are not multiprocessing + assert self.shared_data_choice is not None + assert self.num_processes > 1 + + def get_tally(t): + with self.shared_data_choice_lock: + return self.shared_data_choice[t] + + def wait(tally, target): + while get_tally(tally) != target: + time.sleep(1) + + # - nobody checks in until checkout clears + wait(TALLY_CHECKOUT, 0) + + # - add local_modeled_size data, increment TALLY_CHECKIN + with self.shared_data_choice_lock: + first_in = self.shared_data_choice[TALLY_CHECKIN] == 0 + # add local data from df to shared data buffer + # final column is used for tallys, hence the negative index + # Ellipsis expands : to fill available dims so [..., 0:-1] is the whole array except for the tallys + self.shared_data_choice[..., 0:-1] += local_modeled_size.values.astype( + np.int64 + ) + self.shared_data_choice[TALLY_CHECKIN] += 1 + + # - wait until everybody else has checked in + wait(TALLY_CHECKIN, self.num_processes) + + # - copy shared data, increment TALLY_CHECKIN + with self.shared_data_choice_lock: + logger.info("copy shared_data") + # numpy array with sum of local_modeled_size.values from all processes + global_modeled_size_array = self.shared_data_choice[..., 0:-1].copy() + self.shared_data_choice[TALLY_CHECKOUT] += 1 + + # - first in waits until all other processes have checked out, and cleans tub + if first_in: + wait(TALLY_CHECKOUT, self.num_processes) + with self.shared_data_choice_lock: + # zero shared_data, clear TALLY_CHECKIN, and TALLY_CHECKOUT semaphores + self.shared_data_choice[:] = 0 + logger.info("first_in clearing shared_data") + + # convert summed numpy array data to conform to original dataframe + global_modeled_size_df = pd.DataFrame( + data=global_modeled_size_array, + index=local_modeled_size.index, + columns=local_modeled_size.columns, + ) + + return global_modeled_size_df + def set_choices(self, choices, segment_ids): """ aggregate individual location choices to modeled_size by zone and segment @@ -326,14 +438,32 @@ def set_choices(self, choices, segment_ids): modeled_size[seg_name] = segment_choices.value_counts() + # if self.shadow_price_method == "simulation": + choice_merged = pd.merge( + self.shared_sp_choice_df, + choices, + left_index=True, + right_index=True, + how="left", + suffixes=("_x", "_y"), + ) + + choice_merged["choice_y"] = choice_merged["choice_y"].fillna(0) + choice_merged["choice"] = choice_merged["choice_x"] + choice_merged["choice_y"] + choice_merged = choice_merged.drop(columns=["choice_x", "choice_y"]) + modeled_size = modeled_size.fillna(0).astype(int) if self.num_processes == 1: # - not multiprocessing + self.choices_synced = choices self.modeled_size = modeled_size else: # - if we are multiprocessing, we have to aggregate across sub-processes - self.modeled_size = self.synchronize_choices(modeled_size) + # if self.shadow_price_method == "simulation": + self.choices_synced = self.synchronize_choices(choice_merged) + + self.modeled_size = self.synchronize_modeled_size(modeled_size) def check_fit(self, iteration): """ @@ -368,47 +498,116 @@ def check_fit(self, iteration): # max percentage of zones allowed to fail fail_threshold = self.shadow_settings["FAIL_THRESHOLD"] - modeled_size = self.modeled_size - desired_size = self.desired_size + if self.shadow_settings["SHADOW_PRICE_METHOD"] != "simulation": - abs_diff = (desired_size - modeled_size).abs() + modeled_size = self.modeled_size + desired_size = self.desired_size - rel_diff = abs_diff / modeled_size + abs_diff = (desired_size - modeled_size).abs() - # ignore zones where desired_size < threshold - rel_diff.where(desired_size >= size_threshold, 0, inplace=True) + self.rel_diff = abs_diff / modeled_size - # ignore zones where rel_diff < percent_tolerance - rel_diff.where(rel_diff > (percent_tolerance / 100.0), 0, inplace=True) + # ignore zones where desired_size < threshold + self.rel_diff.where(desired_size >= size_threshold, 0, inplace=True) - self.num_fail["iter%s" % iteration] = (rel_diff > 0).sum() - self.max_abs_diff["iter%s" % iteration] = abs_diff.max() - self.max_rel_diff["iter%s" % iteration] = rel_diff.max() + # ignore zones where rel_diff < percent_tolerance + self.rel_diff.where( + self.rel_diff > (percent_tolerance / 100.0), 0, inplace=True + ) - total_fails = (rel_diff > 0).values.sum() + self.num_fail["iter%s" % iteration] = (self.rel_diff > 0).sum() + self.max_abs_diff["iter%s" % iteration] = abs_diff.max() + self.max_rel_diff["iter%s" % iteration] = self.rel_diff.max() - # FIXME - should not count zones where desired_size < threshold? (could calc in init) - max_fail = (fail_threshold / 100.0) * util.iprod(desired_size.shape) + total_fails = (self.rel_diff > 0).values.sum() - converged = total_fails <= max_fail + # FIXME - should not count zones where desired_size < threshold? (could calc in init) + max_fail = (fail_threshold / 100.0) * util.iprod(desired_size.shape) - # for c in desired_size: - # print("check_fit %s segment %s" % (self.model_selector, c)) - # print(" modeled %s" % (modeled_size[c].sum())) - # print(" desired %s" % (desired_size[c].sum())) - # print(" max abs diff %s" % (abs_diff[c].max())) - # print(" max rel diff %s" % (rel_diff[c].max())) + converged = total_fails <= max_fail - logger.info( - "check_fit %s iteration: %s converged: %s max_fail: %s total_fails: %s" - % (self.model_selector, iteration, converged, max_fail, total_fails) - ) + # for c in desired_size: + # print("check_fit %s segment %s" % (self.model_selector, c)) + # print(" modeled %s" % (modeled_size[c].sum())) + # print(" desired %s" % (desired_size[c].sum())) + # print(" max abs diff %s" % (abs_diff[c].max())) + # print(" max rel diff %s" % (self.rel_diff[c].max())) + + logger.info( + "check_fit %s iteration: %s converged: %s max_fail: %s total_fails: %s" + % (self.model_selector, iteration, converged, max_fail, total_fails) + ) + + # - convergence stats + if converged or iteration == self.max_iterations: + logger.info("\nshadow_pricing max_abs_diff\n%s" % self.max_abs_diff) + logger.info("\nshadow_pricing max_rel_diff\n%s" % self.max_rel_diff) + logger.info("\nshadow_pricing num_fail\n%s" % self.num_fail) + + else: + # ignore convergence criteria for zones smaller than target_threshold + self.target_threshold = self.shadow_settings["TARGET_THRESHOLD"] + + modeled_size = self.modeled_size + desired_size = self.target + + desired_share = self.target / self.target.sum() + modeled_share = ( + self.modeled_size.sum(axis=1) / self.modeled_size.sum().sum() + ) - # - convergence stats - if converged or iteration == self.max_iterations: - logger.info("\nshadow_pricing max_abs_diff\n%s" % self.max_abs_diff) - logger.info("\nshadow_pricing max_rel_diff\n%s" % self.max_rel_diff) - logger.info("\nshadow_pricing num_fail\n%s" % self.num_fail) + self.rel_diff = desired_share / modeled_share + + # abs_diff = (desired_size - modeled_size.sum(axis=1) * (desired_share/modeled_share)).abs() + # abs_diff.to_csv(r'E:\Projects\Clients\SEMCOG\semcog_2zone_rundir\temp\abs_diff.csv') + # rel_diff = abs_diff / (modeled_size.sum(axis=1) * (desired_share/modeled_share)) + + # ignore zones where desired_size < threshold + self.rel_diff.where(desired_size >= self.target_threshold, 0, inplace=True) + + # ignore zones where rel_diff is within percent_tolerance + self.rel_diff.where( + (self.rel_diff > 1 + (percent_tolerance / 100.0)) + | (self.rel_diff < 1 - (percent_tolerance / 100.0)), + 0, + inplace=True, + ) + # self.rel_diff.to_csv(r'E:\Projects\Clients\SEMCOG\semcog_2zone_rundir\temp\rel_diff.csv') + self.num_fail["iter%s" % iteration] = (self.rel_diff > 0).sum() + # self.max_abs_diff["iter%s" % iteration] = abs_diff.max() + # self.max_rel_diff["iter%s" % iteration] = rel_diff.max() + + total_fails = (self.rel_diff > 0).values.sum() + + # FIXME - should not count zones where desired_size < threshold? (could calc in init) + max_fail = (fail_threshold / 100.0) * util.iprod(desired_size.shape) + # print('@@_________@@MAX FAIL') + # print(np.ceil(max_fail)) + # print('failing zones:') + # print(total_fails) + # print('@@_____________@@') + + converged = (total_fails <= np.ceil(max_fail)) | ( + len(self.choices_synced) == 0 + ) + + # for c in desired_size: + # print("check_fit %s segment %s" % (self.model_selector, c)) + # print(" modeled %s" % (modeled_size[c].sum())) + # print(" desired %s" % (desired_size[c].sum())) + # print(" max abs diff %s" % (abs_diff[c].max())) + # print(" max rel diff %s" % (rel_diff[c].max())) + + logger.info( + "check_fit %s iteration: %s converged: %s max_fail: %s total_fails: %s" + % (self.model_selector, iteration, converged, max_fail, total_fails) + ) + + # - convergence stats + if converged or iteration == self.max_iterations: + logger.info("\nshadow_pricing max_abs_diff\n%s" % self.max_abs_diff) + logger.info("\nshadow_pricing max_rel_diff\n%s" % self.max_rel_diff) + logger.info("\nshadow_pricing num_fail\n%s" % self.num_fail) return converged @@ -519,6 +718,74 @@ def update_shadow_prices(self): new_shadow_prices = self.shadow_prices + adjustment + elif shadow_price_method == "simulation": + # - NewMethod + """ + C_j = (emp_j/sum(emp_j))/(workers_j/sum(workers_j)) + + if C_j > 1: #under-estimate workers in zone + + shadow_price_j = 0 + + elif C_j < 1: #over-estimate workers in zone + + shadow_price_j = -999 + resimulate n workers from zone j, with n = int(workers_j-emp_j/sum(emp_j*workers_j)) + """ + + desired_share = self.target / self.target.sum() + modeled_share = ( + self.modeled_size.sum(axis=1) / self.modeled_size.sum().sum() + ) + percent_tolerance = self.shadow_settings["PERCENT_TOLERANCE"] + + sprice = desired_share / modeled_share + sprice.fillna(0, inplace=True) + sprice.replace([np.inf, -np.inf], 0, inplace=True) + + adjustment_ = np.where(sprice <= 1 + percent_tolerance / 100, -999, 0) + adjustment = pd.DataFrame(index=self.shadow_prices.index) + + for seg_id in self.shadow_prices.columns: + adjustment[seg_id] = adjustment_ + + new_shadow_prices = adjustment + self.zonal_sample_rate = 1 - sprice + overpredicted_zones = new_shadow_prices[ + new_shadow_prices.iloc[:, 0] == -999 + ].index + zones_outside_tol = self.zonal_sample_rate[ + self.zonal_sample_rate > percent_tolerance / 100 + ].index + small_zones = self.target[self.target <= self.target_threshold].index + + choices = self.choices_synced[ + (self.choices_synced.choice.isin(overpredicted_zones)) + & (self.choices_synced.choice.isin(zones_outside_tol)) + & ~(self.choices_synced.choice.isin(small_zones)) + ] + + choices_index = choices.index.name + choices = choices.reset_index() + + # handling unlikely cases where there are no more overassigned zones, but a few underassigned zones remain + if len(choices) > 0: + self.sampled_persons = ( + choices.groupby("choice") + .apply( + lambda x: x.sample( + frac=self.zonal_sample_rate.loc[x.name], random_state=1 + ) + ) + .reset_index(drop=True) + .set_index(choices_index) + ) + else: + self.sampled_persons = pd.DataFrame() + + print('_________') + print(len(self.sampled_persons)) + else: raise RuntimeError("unknown SHADOW_PRICE_METHOD %s" % shadow_price_method) @@ -544,10 +811,14 @@ def dest_size_terms(self, segment): size_term_adjustment = self.shadow_prices[segment] elif shadow_price_method == "daysim": utility_adjustment = self.shadow_prices[segment] + elif shadow_price_method == "simulation": + utility_adjustment = self.shadow_prices[segment] else: raise RuntimeError( "unknown SHADOW_PRICE_METHOD %s" % shadow_price_method ) + # added + # utility_adjustment.to_csv(r'C:\Projects\SEMCOG\utility_adjustment.csv') size_terms = pd.DataFrame( { @@ -673,6 +944,126 @@ def buffers_for_shadow_pricing(shadow_pricing_info): return data_buffers +def buffers_for_shadow_pricing_choice(shadow_pricing_choice_info): + """ + Allocate shared_data buffers for multiprocess shadow pricing + + Allocates one buffer per model_selector. + Buffer datatype and shape specified by shadow_pricing_info + + buffers are multiprocessing.Array (RawArray protected by a multiprocessing.Lock wrapper) + We don't actually use the wrapped version as it slows access down and doesn't provide + protection for numpy-wrapped arrays, but it does provide a convenient way to bundle + RawArray and an associated lock. (ShadowPriceCalculator uses the lock to coordinate access to + the numpy-wrapped RawArray.) + + Parameters + ---------- + shadow_pricing_info : dict + + Returns + ------- + data_buffers : dict { : } + dict of multiprocessing.Array keyed by model_selector + """ + + dtype = shadow_pricing_choice_info["dtype"] + block_shapes = shadow_pricing_choice_info["block_shapes"] + + data_buffers = {} + + for block_key, block_shape in block_shapes.items(): + + # buffer_size must be int, not np.int64 + buffer_size = util.iprod(block_shape) + + csz = buffer_size * np.dtype(dtype).itemsize + logger.info( + "allocating shared shadow pricing buffer for choices %s %s buffer_size %s bytes %s (%s)" + % (block_key, buffer_size, block_shape, csz, util.GB(csz)) + ) + + if np.issubdtype(dtype, np.int64): + typecode = ctypes.c_int64 + else: + raise RuntimeError( + "buffer_for_shadow_pricing unrecognized dtype %s" % dtype + ) + + shared_data_buffer = multiprocessing.Array(typecode, buffer_size) + + logger.info("buffer_for_shadow_pricing_choice added block %s" % block_key) + + data_buffers[block_key + "_choice"] = shared_data_buffer + + persons = inject.get_table("persons").to_frame() + sp_choice_df = persons.reset_index()["person_id"].to_frame() + + # declare a shared Array with data from sp_choice_df + mparr = multiprocessing.Array(ctypes.c_double, sp_choice_df.values.reshape(-1)) + + # create a new df based on the shared array + shared_sp_choice_df = pd.DataFrame( + np.frombuffer(mparr.get_obj()).reshape(sp_choice_df.shape), + columns=sp_choice_df.columns, + ) + data_buffers["shadow_price_choice_df"] = shared_sp_choice_df + + return data_buffers + + +def shadow_price_data_from_buffers_choice( + data_buffers, shadow_pricing_info, model_selector +): + """ + + Parameters + ---------- + data_buffers : dict of { : } + multiprocessing.Array is simply a convenient way to bundle Array and Lock + we extract the lock and wrap the RawArray in a numpy array for convenience in indexing + The shared data buffer has shape ( + 1) + extra column is for reverse semaphores with TALLY_CHECKIN and TALLY_CHECKOUT + shadow_pricing_info : dict + dict of useful info + dtype: sp_dtype, + block_shapes : OrderedDict({: }) + dict mapping model_selector to block shape (including extra column for semaphores) + e.g. {'school': (num_zones, num_segments + 1) + model_selector : str + location type model_selector (e.g. school or workplace) + + Returns + ------- + shared_data, shared_data_lock + shared_data : multiprocessing.Array or None (if single process) + shared_data_lock : numpy array wrapping multiprocessing.RawArray or None (if single process) + """ + + assert type(data_buffers) == dict + + dtype = shadow_pricing_info["dtype"] + block_shapes = shadow_pricing_info["block_shapes"] + + if model_selector not in block_shapes: + raise RuntimeError( + "Model selector %s not in shadow_pricing_info" % model_selector + ) + + if block_name(model_selector + "_choice") not in data_buffers: + raise RuntimeError( + "Block %s not in data_buffers" % block_name(model_selector + "_choice") + ) + + data = data_buffers[block_name(model_selector + "_choice")] + shape = ( + int(len(data) / block_shapes[model_selector][1]), + int(block_shapes[model_selector][1]), + ) + + return np.frombuffer(data.get_obj(), dtype=dtype).reshape(shape), data.get_lock() + + def shadow_price_data_from_buffers(data_buffers, shadow_pricing_info, model_selector): """ @@ -747,17 +1138,38 @@ def load_shadow_price_calculator(model_settings): shadow_pricing_info = inject.get_injectable("shadow_pricing_info", None) assert shadow_pricing_info is not None + shadow_pricing_choice_info = inject.get_injectable( + "shadow_pricing_choice_info", None + ) + assert shadow_pricing_choice_info is not None + # - extract data buffer and reshape as numpy array data, lock = shadow_price_data_from_buffers( data_buffers, shadow_pricing_info, model_selector ) + data_choice, lock_choice = shadow_price_data_from_buffers_choice( + data_buffers, shadow_pricing_choice_info, model_selector + ) + if "shadow_price_choice_df" in data_buffers: + shared_sp_choice_df = data_buffers["shadow_price_choice_df"] + else: + shared_sp_choice_df = None + else: assert num_processes == 1 data = None # ShadowPriceCalculator will allocate its own data lock = None # - ShadowPriceCalculator - spc = ShadowPriceCalculator(model_settings, num_processes, data, lock) + spc = ShadowPriceCalculator( + model_settings, + num_processes, + data, + lock, + data_choice, + lock_choice, + shared_sp_choice_df, + ) return spc @@ -918,6 +1330,53 @@ def get_shadow_pricing_info(): return shadow_pricing_info +def get_shadow_pricing_choice_info(): + """ + return dict with info about dtype and shapes of desired and modeled size tables + + block shape is (num_zones, num_segments + 1) + + + Returns + ------- + shadow_pricing_info: dict + dtype: , + block_shapes: dict {: } + """ + + persons = inject.get_table("persons") + # size_terms = inject.get_injectable("size_terms") + + shadow_settings = config.read_model_settings("shadow_pricing.yaml") + + # shadow_pricing_models is dict of {: } + shadow_pricing_models = shadow_settings.get("shadow_pricing_models", {}) + + blocks = OrderedDict() + for model_selector in shadow_pricing_models: + + sp_rows = len(persons) + # sp_cols = len(size_terms[size_terms.model_selector == model_selector]) + + # extra tally column for TALLY_CHECKIN and TALLY_CHECKOUT semaphores + blocks[block_name(model_selector)] = (sp_rows, 2) + + sp_dtype = np.int64 + # sp_dtype = np.str + + shadow_pricing_choice_info = { + "dtype": sp_dtype, + "block_shapes": blocks, + } + + for k in shadow_pricing_choice_info: + logger.debug( + "shadow_pricing_choice_info %s: %s" % (k, shadow_pricing_choice_info.get(k)) + ) + + return shadow_pricing_choice_info + + @inject.injectable(cache=True) def shadow_pricing_info(): @@ -926,3 +1385,35 @@ def shadow_pricing_info(): logger.debug("loading shadow_pricing_info injectable") return get_shadow_pricing_info() + + +@inject.injectable(cache=True) +def shadow_pricing_choice_info(): + + # when multiprocessing with shared data mp_tasks has to call network_los methods + # get_shadow_pricing_info() and buffers_for_shadow_pricing() + logger.debug("loading shadow_pricing_choice_info injectable") + + return get_shadow_pricing_choice_info() + + +# @inject.table() +# def get_full_person_index_df(): + +# persons = inject.get_table('persons').to_frame() +# persons_index = persons.index.name +# persons_index_full = persons.reset_index()[persons_index].reset_index().drop(columns=['index']) + +# inject.add_table('persons_index_full', persons_index_full) +# #pipeline.get_rn_generator().add_channel('persons_index_full', persons_index_full) + +# return persons_index_full + +# @inject.injectable(cache=True) +# def get_full_person_index(): + +# persons = inject.get_table('persons').to_frame() +# persons_index_full = persons.index + +# return persons_index_full + diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 1d41c7c96..9ae776193 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -1056,6 +1056,33 @@ def allocate_shared_shadow_pricing_buffers(): return shadow_pricing_buffers +def allocate_shared_shadow_pricing_buffers_choice(): + """ + This is called by the main process to allocate memory buffer to share with subprocs + + Returns + ------- + multiprocessing.RawArray + """ + + info("allocate_shared_shadow_pricing_buffers_choice") + + shadow_pricing_choice_info = inject.get_injectable( + "shadow_pricing_choice_info", None + ) + + if shadow_pricing_choice_info is not None: + from activitysim.abm.tables import shadow_pricing + + shadow_pricing_buffers_choice = shadow_pricing.buffers_for_shadow_pricing_choice( + shadow_pricing_choice_info + ) + else: + shadow_pricing_buffers_choice = {} + + return shadow_pricing_buffers_choice + + def run_sub_simulations( injectables, shared_data_buffers, @@ -1401,6 +1428,13 @@ def find_breadcrumb(crumb, default=None): t0 = tracing.print_elapsed_time("allocate shared shadow_pricing buffer", t0) mem.trace_memory_info("allocate_shared_shadow_pricing_buffers.completed") + # combine shared_shadow_pricing_buffers to pool choices across all processes + t0 = tracing.print_elapsed_time() + shared_data_buffers.update(allocate_shared_shadow_pricing_buffers_choice()) + t0 = tracing.print_elapsed_time("allocate shared shadow_pricing choice buffer", t0) + mem.trace_memory_info("allocate_shared_shadow_pricing_buffers_choice.completed") + + # print(shared_data_buffers) # - mp_setup_skims if len(shared_data_buffers) > 0: run_sub_task( From 11600c37079fb9adf9fd9cd427db555b0c8a5696 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 30 Sep 2022 13:12:30 -0700 Subject: [PATCH 02/27] blacken --- activitysim/abm/models/location_choice.py | 8 +++++++- activitysim/abm/tables/shadow_pricing.py | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 330e669d4..4cd928968 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -973,7 +973,13 @@ def iterate_location_choice( spc.write_trace_files(iteration) if spc.use_shadow_pricing and spc.check_fit(iteration): - logging.info("%s converged after iteration %s" % (trace_label, iteration,)) + logging.info( + "%s converged after iteration %s" + % ( + trace_label, + iteration, + ) + ) break # - shadow price table diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 61a4e29a1..84f83bdb9 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -783,7 +783,7 @@ def update_shadow_prices(self): else: self.sampled_persons = pd.DataFrame() - print('_________') + print("_________") print(len(self.sampled_persons)) else: @@ -1416,4 +1416,3 @@ def shadow_pricing_choice_info(): # persons_index_full = persons.index # return persons_index_full - From 4b9f55edf9d9f7411f5537d55e23479e86213e7b Mon Sep 17 00:00:00 2001 From: aletzdy <58451076+aletzdy@users.noreply.github.com> Date: Fri, 30 Sep 2022 13:28:53 -0700 Subject: [PATCH 03/27] Updated shadow_pricing.yaml for mtc example --- .../prototype_mtc/configs/shadow_pricing.yaml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml b/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml index b61ec4192..8351b5fc2 100644 --- a/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml +++ b/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml @@ -4,7 +4,7 @@ shadow_pricing_models: # global switch to enable/disable loading of saved shadow prices # (ignored if global use_shadow_pricing switch is False) -LOAD_SAVED_SHADOW_PRICES: True +LOAD_SAVED_SHADOW_PRICES: False # number of shadow price iterations for cold start MAX_ITERATIONS: 10 @@ -15,15 +15,23 @@ MAX_ITERATIONS_SAVED: 1 # ignore criteria for zones smaller than size_threshold SIZE_THRESHOLD: 10 +# simulation method: ignore criteria for zones smaller than target_threshold (total employmnet or enrollment) +TARGET_THRESHOLD: 20 + # zone passes if modeled is within percent_tolerance of predicted_size PERCENT_TOLERANCE: 5 # max percentage of zones allowed to fail -FAIL_THRESHOLD: 10 +FAIL_THRESHOLD: 1 + +## Shadow pricing method +# SHADOW_PRICE_METHOD: ctramp +# SHADOW_PRICE_METHOD: daysim +SHADOW_PRICE_METHOD: simulation -# CTRAMP or daysim -SHADOW_PRICE_METHOD: ctramp -#SHADOW_PRICE_METHOD: daysim +# simulation method: total employment/enrollment fields in landuse +TOTAL_EMP: TOTEMP +TOTAL_ENORLLMENT: TOTENR # ctramp-style shadow_pricing_method parameters DAMPING_FACTOR: 1 @@ -32,3 +40,4 @@ DAMPING_FACTOR: 1 # FIXME should these be the same as PERCENT_TOLERANCE and FAIL_THRESHOLD above? DAYSIM_ABSOLUTE_TOLERANCE: 50 DAYSIM_PERCENT_TOLERANCE: 10 + From 846d3be1afebf12d626339033353f9ab60dc007e Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 30 Sep 2022 15:51:12 -0700 Subject: [PATCH 04/27] code cleanup --- activitysim/abm/tables/shadow_pricing.py | 82 +++---------------- .../prototype_mtc/configs/shadow_pricing.yaml | 76 ++++++++--------- .../configs/annotate_landuse.csv | 7 ++ .../configs/shadow_pricing.yaml | 40 +++++++++ .../configs_mp/settings.yaml | 4 +- 5 files changed, 94 insertions(+), 115 deletions(-) create mode 100644 activitysim/examples/prototype_mtc_extended/configs/annotate_landuse.csv create mode 100644 activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 84f83bdb9..7c82e9996 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -82,7 +82,6 @@ def __init__( shared_sp_choice_df=None, ): """ - Presence of shared_data is used as a flag for multiprocessing If we are multiprocessing, shared_data should be a multiprocessing.RawArray buffer to aggregate modeled_size across all sub-processes, and shared_data_lock should be @@ -158,9 +157,10 @@ def __init__( self.shared_data_choice_lock = shared_data_choice_lock self.shared_sp_choice_df = shared_sp_choice_df - self.shared_sp_choice_df = self.shared_sp_choice_df.astype("int") - self.shared_sp_choice_df = self.shared_sp_choice_df.set_index("person_id") - self.shared_sp_choice_df["choice"] = int(0) + if shared_sp_choice_df is not None: + self.shared_sp_choice_df = self.shared_sp_choice_df.astype("int") + self.shared_sp_choice_df = self.shared_sp_choice_df.set_index("person_id") + self.shared_sp_choice_df["choice"] = int(0) # - load saved shadow_prices (if available) and set max_iterations accordingly if self.use_shadow_pricing: @@ -210,7 +210,7 @@ def __init__( self.target = land_use[self.total_emp] elif self.model_selector == "school": - total_enr = self.shadow_settings["TOTAL_ENORLLMENT"] + total_enr = self.shadow_settings["TOTAL_ENROLLMENT"] land_use = inject.get_table("land_use").to_frame() self.target = land_use[total_enr] self.zonal_sample_rate = None @@ -526,13 +526,6 @@ def check_fit(self, iteration): converged = total_fails <= max_fail - # for c in desired_size: - # print("check_fit %s segment %s" % (self.model_selector, c)) - # print(" modeled %s" % (modeled_size[c].sum())) - # print(" desired %s" % (desired_size[c].sum())) - # print(" max abs diff %s" % (abs_diff[c].max())) - # print(" max rel diff %s" % (self.rel_diff[c].max())) - logger.info( "check_fit %s iteration: %s converged: %s max_fail: %s total_fails: %s" % (self.model_selector, iteration, converged, max_fail, total_fails) @@ -558,10 +551,6 @@ def check_fit(self, iteration): self.rel_diff = desired_share / modeled_share - # abs_diff = (desired_size - modeled_size.sum(axis=1) * (desired_share/modeled_share)).abs() - # abs_diff.to_csv(r'E:\Projects\Clients\SEMCOG\semcog_2zone_rundir\temp\abs_diff.csv') - # rel_diff = abs_diff / (modeled_size.sum(axis=1) * (desired_share/modeled_share)) - # ignore zones where desired_size < threshold self.rel_diff.where(desired_size >= self.target_threshold, 0, inplace=True) @@ -572,7 +561,6 @@ def check_fit(self, iteration): 0, inplace=True, ) - # self.rel_diff.to_csv(r'E:\Projects\Clients\SEMCOG\semcog_2zone_rundir\temp\rel_diff.csv') self.num_fail["iter%s" % iteration] = (self.rel_diff > 0).sum() # self.max_abs_diff["iter%s" % iteration] = abs_diff.max() # self.max_rel_diff["iter%s" % iteration] = rel_diff.max() @@ -581,23 +569,11 @@ def check_fit(self, iteration): # FIXME - should not count zones where desired_size < threshold? (could calc in init) max_fail = (fail_threshold / 100.0) * util.iprod(desired_size.shape) - # print('@@_________@@MAX FAIL') - # print(np.ceil(max_fail)) - # print('failing zones:') - # print(total_fails) - # print('@@_____________@@') converged = (total_fails <= np.ceil(max_fail)) | ( len(self.choices_synced) == 0 ) - # for c in desired_size: - # print("check_fit %s segment %s" % (self.model_selector, c)) - # print(" modeled %s" % (modeled_size[c].sum())) - # print(" desired %s" % (desired_size[c].sum())) - # print(" max abs diff %s" % (abs_diff[c].max())) - # print(" max rel diff %s" % (rel_diff[c].max())) - logger.info( "check_fit %s iteration: %s converged: %s max_fail: %s total_fails: %s" % (self.model_selector, iteration, converged, max_fail, total_fails) @@ -783,17 +759,9 @@ def update_shadow_prices(self): else: self.sampled_persons = pd.DataFrame() - print("_________") - print(len(self.sampled_persons)) - else: raise RuntimeError("unknown SHADOW_PRICE_METHOD %s" % shadow_price_method) - # print("\nself.desired_size\n%s" % self.desired_size.head()) - # print("\nself.modeled_size\n%s" % self.modeled_size.head()) - # print("\nprevious shadow_prices\n%s" % self.shadow_prices.head()) - # print("\nnew_shadow_prices\n%s" % new_shadow_prices.head()) - self.shadow_prices = new_shadow_prices def dest_size_terms(self, segment): @@ -817,8 +785,6 @@ def dest_size_terms(self, segment): raise RuntimeError( "unknown SHADOW_PRICE_METHOD %s" % shadow_price_method ) - # added - # utility_adjustment.to_csv(r'C:\Projects\SEMCOG\utility_adjustment.csv') size_terms = pd.DataFrame( { @@ -946,25 +912,20 @@ def buffers_for_shadow_pricing(shadow_pricing_info): def buffers_for_shadow_pricing_choice(shadow_pricing_choice_info): """ - Allocate shared_data buffers for multiprocess shadow pricing - - Allocates one buffer per model_selector. - Buffer datatype and shape specified by shadow_pricing_info - - buffers are multiprocessing.Array (RawArray protected by a multiprocessing.Lock wrapper) - We don't actually use the wrapped version as it slows access down and doesn't provide - protection for numpy-wrapped arrays, but it does provide a convenient way to bundle - RawArray and an associated lock. (ShadowPriceCalculator uses the lock to coordinate access to - the numpy-wrapped RawArray.) + Same as above buffers_for_shadow_price function except now we need to store + the actual choices for the simulation based shadow pricing method + This allocates a multiprocessing.Array that can store the choice for each person + and then wraps a dataframe around it. That means the dataframe can be shared + and accessed across all threads. Parameters ---------- shadow_pricing_info : dict - Returns ------- data_buffers : dict { : } dict of multiprocessing.Array keyed by model_selector + and wrapped in a pandas dataframe """ dtype = shadow_pricing_choice_info["dtype"] @@ -1395,24 +1356,3 @@ def shadow_pricing_choice_info(): logger.debug("loading shadow_pricing_choice_info injectable") return get_shadow_pricing_choice_info() - - -# @inject.table() -# def get_full_person_index_df(): - -# persons = inject.get_table('persons').to_frame() -# persons_index = persons.index.name -# persons_index_full = persons.reset_index()[persons_index].reset_index().drop(columns=['index']) - -# inject.add_table('persons_index_full', persons_index_full) -# #pipeline.get_rn_generator().add_channel('persons_index_full', persons_index_full) - -# return persons_index_full - -# @inject.injectable(cache=True) -# def get_full_person_index(): - -# persons = inject.get_table('persons').to_frame() -# persons_index_full = persons.index - -# return persons_index_full diff --git a/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml b/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml index 8351b5fc2..878812baf 100644 --- a/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml +++ b/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml @@ -1,43 +1,35 @@ shadow_pricing_models: - school: school_location - workplace: workplace_location - -# global switch to enable/disable loading of saved shadow prices -# (ignored if global use_shadow_pricing switch is False) -LOAD_SAVED_SHADOW_PRICES: False - -# number of shadow price iterations for cold start -MAX_ITERATIONS: 10 - -# number of shadow price iterations for warm start (after loading saved shadow_prices) -MAX_ITERATIONS_SAVED: 1 - -# ignore criteria for zones smaller than size_threshold -SIZE_THRESHOLD: 10 - -# simulation method: ignore criteria for zones smaller than target_threshold (total employmnet or enrollment) -TARGET_THRESHOLD: 20 - -# zone passes if modeled is within percent_tolerance of predicted_size -PERCENT_TOLERANCE: 5 - -# max percentage of zones allowed to fail -FAIL_THRESHOLD: 1 - -## Shadow pricing method -# SHADOW_PRICE_METHOD: ctramp -# SHADOW_PRICE_METHOD: daysim -SHADOW_PRICE_METHOD: simulation - -# simulation method: total employment/enrollment fields in landuse -TOTAL_EMP: TOTEMP -TOTAL_ENORLLMENT: TOTENR - -# ctramp-style shadow_pricing_method parameters -DAMPING_FACTOR: 1 - -# daysim-style shadow_pricing_method parameters -# FIXME should these be the same as PERCENT_TOLERANCE and FAIL_THRESHOLD above? -DAYSIM_ABSOLUTE_TOLERANCE: 50 -DAYSIM_PERCENT_TOLERANCE: 10 - + school: school_location + workplace: workplace_location + + # global switch to enable/disable loading of saved shadow prices + # (ignored if global use_shadow_pricing switch is False) + LOAD_SAVED_SHADOW_PRICES: True + + # number of shadow price iterations for cold start + MAX_ITERATIONS: 10 + + # number of shadow price iterations for warm start (after loading saved shadow_prices) + MAX_ITERATIONS_SAVED: 1 + + # ignore criteria for zones smaller than size_threshold + SIZE_THRESHOLD: 10 + + # zone passes if modeled is within percent_tolerance of predicted_size + PERCENT_TOLERANCE: 5 + + # max percentage of zones allowed to fail + FAIL_THRESHOLD: 10 + + # CTRAMP or daysim + SHADOW_PRICE_METHOD: ctramp + #SHADOW_PRICE_METHOD: daysim + + # ctramp-style shadow_pricing_method parameters + DAMPING_FACTOR: 1 + + # daysim-style shadow_pricing_method parameters + # FIXME should these be the same as PERCENT_TOLERANCE and FAIL_THRESHOLD above? + DAYSIM_ABSOLUTE_TOLERANCE: 50 + DAYSIM_PERCENT_TOLERANCE: 10 + \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc_extended/configs/annotate_landuse.csv b/activitysim/examples/prototype_mtc_extended/configs/annotate_landuse.csv new file mode 100644 index 000000000..9b1dd5eb6 --- /dev/null +++ b/activitysim/examples/prototype_mtc_extended/configs/annotate_landuse.csv @@ -0,0 +1,7 @@ +Description,Target,Expression +#,, annotate landuse table after import +household_density,household_density,land_use.TOTHH / (land_use.RESACRE + land_use.CIACRE) +employment_density,employment_density,land_use.TOTEMP / (land_use.RESACRE + land_use.CIACRE) +density_index,density_index,(household_density *employment_density) / (household_density + employment_density).clip(lower=1) +,is_cbd,land_use.area_type == 1 +total enrollment for simulation based shadow pricing,TOTENR,land_use.AGE0519 + land_use.HSENROLL + land_use.COLLFTE + land_use.COLLPTE \ No newline at end of file diff --git a/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml b/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml new file mode 100644 index 000000000..e314e918e --- /dev/null +++ b/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml @@ -0,0 +1,40 @@ +shadow_pricing_models: + school: school_location + workplace: workplace_location + +# global switch to enable/disable loading of saved shadow prices +# (ignored if global use_shadow_pricing switch is False) +LOAD_SAVED_SHADOW_PRICES: False + +# number of shadow price iterations for cold start +MAX_ITERATIONS: 10 + +# number of shadow price iterations for warm start (after loading saved shadow_prices) +MAX_ITERATIONS_SAVED: 1 + +## Shadow pricing method +# SHADOW_PRICE_METHOD: ctramp +# SHADOW_PRICE_METHOD: daysim +SHADOW_PRICE_METHOD: simulation + +# --- simulation method settings +# total employment/enrollment fields in landuse +TOTAL_EMP: TOTEMP +TOTAL_ENROLLMENT: TOTENR +# ignore criteria for zones smaller than size_threshold +SIZE_THRESHOLD: 10 +# ignore criteria for zones smaller than target_threshold (total employmnet or enrollment) +TARGET_THRESHOLD: 20 +# zone passes if modeled is within percent_tolerance of predicted_size +PERCENT_TOLERANCE: 5 +# max percentage of zones allowed to fail +FAIL_THRESHOLD: 1 + +# --- ctramp method settings +# DAMPING_FACTOR: 1 + +# --- daysim method settings +# FIXME should these be the same as PERCENT_TOLERANCE and FAIL_THRESHOLD above? +# DAYSIM_ABSOLUTE_TOLERANCE: 50 +# DAYSIM_PERCENT_TOLERANCE: 10 + diff --git a/activitysim/examples/prototype_mtc_extended/configs_mp/settings.yaml b/activitysim/examples/prototype_mtc_extended/configs_mp/settings.yaml index 9938537c7..0e6aa415e 100644 --- a/activitysim/examples/prototype_mtc_extended/configs_mp/settings.yaml +++ b/activitysim/examples/prototype_mtc_extended/configs_mp/settings.yaml @@ -20,7 +20,7 @@ fail_fast: True # - ------------------------- dev config multiprocess: True strict: False -use_shadow_pricing: False +use_shadow_pricing: True households_sample_size: 0 chunk_size: 0 @@ -35,7 +35,7 @@ want_dest_choice_sample_tables: False #write_skim_cache: True # - tracing -#trace_hh_id: +trace_hh_id: trace_od: # to resume after last successful checkpoint, specify resume_after: _ From e2dde113d54c3fca05c2e1e3d63e9206e1a36ce4 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 30 Sep 2022 16:15:57 -0700 Subject: [PATCH 05/27] more cleanup --- activitysim/abm/models/location_choice.py | 4 ---- activitysim/core/mp_tasks.py | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 4cd928968..ef09617e9 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -886,7 +886,6 @@ def iterate_location_choice( for iteration in range(1, max_iterations + 1): persons_merged_df_ = persons_merged_df.copy() - # print("################################_" + str(iteration)) if spc.use_shadow_pricing and iteration > 1: spc.update_shadow_prices() @@ -916,9 +915,6 @@ def iterate_location_choice( persons_merged_df_ = persons_merged_df_.sort_index() - # sample_df["n_" + str(iteration)] = spc.sampled_persons - # sample_df.to_csv(r"E:\Projects\Clients\SEMCOG\semcog_2zone_rundir\temp\sample_df" + ".csv") - choices_df_, save_sample_df = run_location_choice( persons_merged_df_, network_los, diff --git a/activitysim/core/mp_tasks.py b/activitysim/core/mp_tasks.py index 9ae776193..429ff2e86 100644 --- a/activitysim/core/mp_tasks.py +++ b/activitysim/core/mp_tasks.py @@ -1074,8 +1074,8 @@ def allocate_shared_shadow_pricing_buffers_choice(): if shadow_pricing_choice_info is not None: from activitysim.abm.tables import shadow_pricing - shadow_pricing_buffers_choice = shadow_pricing.buffers_for_shadow_pricing_choice( - shadow_pricing_choice_info + shadow_pricing_buffers_choice = ( + shadow_pricing.buffers_for_shadow_pricing_choice(shadow_pricing_choice_info) ) else: shadow_pricing_buffers_choice = {} @@ -1434,7 +1434,6 @@ def find_breadcrumb(crumb, default=None): t0 = tracing.print_elapsed_time("allocate shared shadow_pricing choice buffer", t0) mem.trace_memory_info("allocate_shared_shadow_pricing_buffers_choice.completed") - # print(shared_data_buffers) # - mp_setup_skims if len(shared_data_buffers) > 0: run_sub_task( From 38633961222e7c899c3605cdc249a9dc1b9dbff1 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 30 Sep 2022 16:26:33 -0700 Subject: [PATCH 06/27] documentation and passing tests --- activitysim/abm/tables/shadow_pricing.py | 37 +++++-------------- .../prototype_mtc/configs/shadow_pricing.yaml | 4 +- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 7c82e9996..10f0a94a3 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -19,9 +19,9 @@ See docstrings for documentation on: -update_shadow_prices how shadow_price coefficients are calculated -synchronize_choices interprocess communication to compute aggregate modeled_size -check_fit convergence criteria for shadow_pric iteration +update_shadow_prices how shadow_price coefficients are calculated +synchronize_modeled_size interprocess communication to compute aggregate modeled_size +check_fit convergence criteria for shadow_pric iteration Import concepts and variables: @@ -46,7 +46,7 @@ we use the first two rows of the final column in numpy-wrapped shared data as 'reverse semaphores' (they synchronize concurrent access to shared data resource rather than throttling access) -ShadowPriceCalculator.synchronize_choices coordinates access to the global aggregate zone counts +ShadowPriceCalculator.synchronize_modeled_size coordinates access to the global aggregate zone counts (local_modeled_size summed across all sub-processes) using these two semaphores (which are really only tuples of indexes of locations in the shared data array. """ @@ -122,7 +122,7 @@ def __init__( self.segment_ids = model_settings["SEGMENT_IDS"] - # - modeled_size (set by call to set_choices/synchronize_choices) + # - modeled_size (set by call to set_choices/synchronize_modeled_size) self.modeled_size = None if self.use_shadow_pricing: @@ -256,7 +256,7 @@ def synchronize_modeled_size(self, local_modeled_size): zone counts are in shared data, we have to coordinate access to the data structure across sub-processes. Note that all access to self.shared_data has to be protected by acquiring shared_data_lock - ShadowPriceCalculator.synchronize_choices coordinates access to the global aggregate + ShadowPriceCalculator.synchronize_modeled_size coordinates access to the global aggregate zone counts (local_modeled_size summed across all sub-processes). * All processes wait (in case we are iterating) until any stragglers from the previous iteration have exited the building. (TALLY_CHECKOUT goes to zero) @@ -328,27 +328,9 @@ def wait(tally, target): def synchronize_choices(self, local_modeled_size): """ - We have to wait until all processes have computed choices and aggregated them by segment - and zone before we can compute global aggregate zone counts (by segment). Since the global - zone counts are in shared data, we have to coordinate access to the data structure across - sub-processes. - - Note that all access to self.shared_data has to be protected by acquiring shared_data_lock - - ShadowPriceCalculator.synchronize_choices coordinates access to the global aggregate - zone counts (local_modeled_size summed across all sub-processes). - - * All processes wait (in case we are iterating) until any stragglers from the previous - iteration have exited the building. (TALLY_CHECKOUT goes to zero) - - * Processes then add their local counts into the shared_data and increment TALLY_CHECKIN - - * All processes wait until everybody has checked in (TALLY_CHECKIN == num_processes) - - * Processes make local copy of shared_data and check out (increment TALLY_CHECKOUT) - - * first_in process waits until all processes have checked out, then zeros shared_data - and clears semaphores + Same thing as the above synchronize_modeled_size method with the small + difference of keeping track of the individual choices instead of the + aggregate modeled choices between processes. Parameters ---------- @@ -460,7 +442,6 @@ def set_choices(self, choices, segment_ids): self.modeled_size = modeled_size else: # - if we are multiprocessing, we have to aggregate across sub-processes - # if self.shadow_price_method == "simulation": self.choices_synced = self.synchronize_choices(choice_merged) self.modeled_size = self.synchronize_modeled_size(modeled_size) diff --git a/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml b/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml index 878812baf..ca16b74ee 100644 --- a/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml +++ b/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml @@ -1,6 +1,6 @@ shadow_pricing_models: - school: school_location - workplace: workplace_location + school: school_location + workplace: workplace_location # global switch to enable/disable loading of saved shadow prices # (ignored if global use_shadow_pricing switch is False) From 3d1edc7529280f40d0b42f07d99192ed1b2ba571 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 30 Sep 2022 16:33:41 -0700 Subject: [PATCH 07/27] passing tests --- activitysim/abm/tables/shadow_pricing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 10f0a94a3..a18c8a571 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -329,7 +329,7 @@ def wait(tally, target): def synchronize_choices(self, local_modeled_size): """ Same thing as the above synchronize_modeled_size method with the small - difference of keeping track of the individual choices instead of the + difference of keeping track of the individual choices instead of the aggregate modeled choices between processes. Parameters From c4dc4eadd10a39245ec689833e8d4b9a1cef358b Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 30 Sep 2022 18:07:57 -0700 Subject: [PATCH 08/27] passing tests --- activitysim/abm/tables/shadow_pricing.py | 38 +++++++----- .../prototype_mtc/configs/shadow_pricing.yaml | 60 +++++++++---------- 2 files changed, 52 insertions(+), 46 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index a18c8a571..c5fd0efb1 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -420,20 +420,6 @@ def set_choices(self, choices, segment_ids): modeled_size[seg_name] = segment_choices.value_counts() - # if self.shadow_price_method == "simulation": - choice_merged = pd.merge( - self.shared_sp_choice_df, - choices, - left_index=True, - right_index=True, - how="left", - suffixes=("_x", "_y"), - ) - - choice_merged["choice_y"] = choice_merged["choice_y"].fillna(0) - choice_merged["choice"] = choice_merged["choice_x"] + choice_merged["choice_y"] - choice_merged = choice_merged.drop(columns=["choice_x", "choice_y"]) - modeled_size = modeled_size.fillna(0).astype(int) if self.num_processes == 1: @@ -442,10 +428,27 @@ def set_choices(self, choices, segment_ids): self.modeled_size = modeled_size else: # - if we are multiprocessing, we have to aggregate across sub-processes - self.choices_synced = self.synchronize_choices(choice_merged) - self.modeled_size = self.synchronize_modeled_size(modeled_size) + # need to also store individual choices if simulation approach + if self.shadow_price_method == "simulation": + choice_merged = pd.merge( + self.shared_sp_choice_df, + choices, + left_index=True, + right_index=True, + how="left", + suffixes=("_x", "_y"), + ) + + choice_merged["choice_y"] = choice_merged["choice_y"].fillna(0) + choice_merged["choice"] = ( + choice_merged["choice_x"] + choice_merged["choice_y"] + ) + choice_merged = choice_merged.drop(columns=["choice_x", "choice_y"]) + + self.choices_synced = self.synchronize_choices(choice_merged) + def check_fit(self, iteration): """ Check convergence criteria fit of modeled_size to target desired_size @@ -1101,6 +1104,9 @@ def load_shadow_price_calculator(model_settings): assert num_processes == 1 data = None # ShadowPriceCalculator will allocate its own data lock = None + data_choice = None + lock_choice = None + shared_sp_choice_df = None # - ShadowPriceCalculator spc = ShadowPriceCalculator( diff --git a/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml b/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml index ca16b74ee..89816475a 100644 --- a/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml +++ b/activitysim/examples/prototype_mtc/configs/shadow_pricing.yaml @@ -2,34 +2,34 @@ shadow_pricing_models: school: school_location workplace: workplace_location - # global switch to enable/disable loading of saved shadow prices - # (ignored if global use_shadow_pricing switch is False) - LOAD_SAVED_SHADOW_PRICES: True - - # number of shadow price iterations for cold start - MAX_ITERATIONS: 10 - - # number of shadow price iterations for warm start (after loading saved shadow_prices) - MAX_ITERATIONS_SAVED: 1 - - # ignore criteria for zones smaller than size_threshold - SIZE_THRESHOLD: 10 - - # zone passes if modeled is within percent_tolerance of predicted_size - PERCENT_TOLERANCE: 5 - - # max percentage of zones allowed to fail - FAIL_THRESHOLD: 10 - - # CTRAMP or daysim - SHADOW_PRICE_METHOD: ctramp - #SHADOW_PRICE_METHOD: daysim - - # ctramp-style shadow_pricing_method parameters - DAMPING_FACTOR: 1 - - # daysim-style shadow_pricing_method parameters - # FIXME should these be the same as PERCENT_TOLERANCE and FAIL_THRESHOLD above? - DAYSIM_ABSOLUTE_TOLERANCE: 50 - DAYSIM_PERCENT_TOLERANCE: 10 +# global switch to enable/disable loading of saved shadow prices +# (ignored if global use_shadow_pricing switch is False) +LOAD_SAVED_SHADOW_PRICES: True + +# number of shadow price iterations for cold start +MAX_ITERATIONS: 10 + +# number of shadow price iterations for warm start (after loading saved shadow_prices) +MAX_ITERATIONS_SAVED: 1 + +# ignore criteria for zones smaller than size_threshold +SIZE_THRESHOLD: 10 + +# zone passes if modeled is within percent_tolerance of predicted_size +PERCENT_TOLERANCE: 5 + +# max percentage of zones allowed to fail +FAIL_THRESHOLD: 10 + +# CTRAMP or daysim +SHADOW_PRICE_METHOD: ctramp +#SHADOW_PRICE_METHOD: daysim + +# ctramp-style shadow_pricing_method parameters +DAMPING_FACTOR: 1 + +# daysim-style shadow_pricing_method parameters +# FIXME should these be the same as PERCENT_TOLERANCE and FAIL_THRESHOLD above? +DAYSIM_ABSOLUTE_TOLERANCE: 50 +DAYSIM_PERCENT_TOLERANCE: 10 \ No newline at end of file From 23df7b9f8ced88df236bb6178928378fd1b4b001 Mon Sep 17 00:00:00 2001 From: aletzdy <58451076+aletzdy@users.noreply.github.com> Date: Mon, 3 Oct 2022 09:04:43 -0400 Subject: [PATCH 09/27] updated doc on shadow pricing --- docs/models.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/models.rst b/docs/models.rst index 00dd28aa4..bd4ac0e63 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -206,17 +206,18 @@ for iteration 3 of the school location model will be called ``trace.shadow_price In total, ActivitySim generates three types of output files for each model with shadow pricing: - ``trace.shadow_price__desired_size.csv`` - The size terms by zone that shadow pricing is attempting to target. These usually will match the size terms identified - in the land_use input file. + The size terms by zone that the ctramp and daysim methods are attempting to target. These equal the size term columns in the land use data multiplied by size term coefficients. - ``trace.shadow_price__modeled_size_.csv`` These are the modeled size terms after the iteration of shadow pricing identified by the number. In other - words, these are the predicted choices by zone for the model after the iteration completes. + words, these are the predicted choices by zone and segment for the model after the iteration completes. - ``trace.shadow_price__shadow_prices_.csv`` - The actual shadow price for each zone and segment after the of shadow pricing. This the file that can be + The actual shadow price for each zone and segment after the of shadow pricing. This is the file that can be used to warm start the shadow pricing mechanism in ActivitySim. +There are three shadow pricing methods in activitysim: ``ctramp``, ``daysim``, and ``simulation``. The first two methods try to match model output with workplace/school location model size terms, while the last method matches model output with actual employment/enrolmment data. + **shadow_pricing.yaml Attributes** - ``shadow_pricing_models`` List model_selectors and model_names of models that use shadow pricing. This list identifies which size_terms to preload which must be done in single process mode, so predicted_size tables can be scaled to population) @@ -225,8 +226,8 @@ In total, ActivitySim generates three types of output files for each model with - ``MAX_ITERATIONS_SAVED`` If loaded shadow prices, maximum number of times shadow pricing can be run. - ``SIZE_THRESHOLD`` Ignore zones in failure calculation with fewer choices than specified here. - ``PERCENT_TOLERANCE`` Maximum percent difference between modeled and desired size terms -- ``FAIL_THRESHOLD`` Number of zones exceeding the PERCENT_TOLERANCE considered a failure -- ``SHADOW_PRICE_METHOD`` [ctramp | daysim] +- ``FAIL_THRESHOLD`` percentage of zones exceeding the PERCENT_TOLERANCE considered a failure +- ``SHADOW_PRICE_METHOD`` [ctramp | daysim | simulation] - ``DAMPING_FACTOR`` On each iteration, ActivitySim will attempt to adjust the model to match desired size terms. The number is multiplied by adjustment factor to dampen or amplify the ActivitySim calculation. (only for CT-RAMP) - ``DAYSIM_ABSOLUTE_TOLERANCE`` From 4269dd554f2d72f2903402009ba1f1c4ccf8178c Mon Sep 17 00:00:00 2001 From: aletzdy <58451076+aletzdy@users.noreply.github.com> Date: Mon, 3 Oct 2022 09:28:07 -0400 Subject: [PATCH 10/27] 2nd Update model doc on shadow pricing --- docs/models.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/models.rst b/docs/models.rst index bd4ac0e63..3eba59d7a 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -224,10 +224,13 @@ There are three shadow pricing methods in activitysim: ``ctramp``, ``daysim``, a - ``LOAD_SAVED_SHADOW_PRICES`` global switch to enable/disable loading of saved shadow prices. From the above example, this would be trace.shadow_price__shadow_prices_.csv renamed and stored in the ``data_dir``. - ``MAX_ITERATIONS`` If no loaded shadow prices, maximum number of times shadow pricing can be run on each model before proceeding to the next model. - ``MAX_ITERATIONS_SAVED`` If loaded shadow prices, maximum number of times shadow pricing can be run. -- ``SIZE_THRESHOLD`` Ignore zones in failure calculation with fewer choices than specified here. +- ``SIZE_THRESHOLD`` Ignore zones in failure calculation (ctramp or daysim method) with smaller size term value than size_threshold. +- ``TARGET_THRESHOLD`` Ignore zones in failure calculation (simulation method) with smaller employment/enrollment than target_threshold. - ``PERCENT_TOLERANCE`` Maximum percent difference between modeled and desired size terms - ``FAIL_THRESHOLD`` percentage of zones exceeding the PERCENT_TOLERANCE considered a failure - ``SHADOW_PRICE_METHOD`` [ctramp | daysim | simulation] +- ``TOTAL_EMP`` total employment column name defined in the land use data +- ``TOTAL_ENORLLMENT``: total enrolmment colmns name defined in the land use data - ``DAMPING_FACTOR`` On each iteration, ActivitySim will attempt to adjust the model to match desired size terms. The number is multiplied by adjustment factor to dampen or amplify the ActivitySim calculation. (only for CT-RAMP) - ``DAYSIM_ABSOLUTE_TOLERANCE`` From 3a87acdc14e726bafb96e825650773d29b16bb5c Mon Sep 17 00:00:00 2001 From: aletzdy <58451076+aletzdy@users.noreply.github.com> Date: Mon, 3 Oct 2022 09:31:34 -0400 Subject: [PATCH 11/27] more doc update on shadow pricing --- docs/models.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/models.rst b/docs/models.rst index 3eba59d7a..200504694 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -216,7 +216,7 @@ In total, ActivitySim generates three types of output files for each model with The actual shadow price for each zone and segment after the of shadow pricing. This is the file that can be used to warm start the shadow pricing mechanism in ActivitySim. -There are three shadow pricing methods in activitysim: ``ctramp``, ``daysim``, and ``simulation``. The first two methods try to match model output with workplace/school location model size terms, while the last method matches model output with actual employment/enrolmment data. +There are three shadow pricing methods in activitysim: ``ctramp``, ``daysim``, and ``simulation``. The first two methods try to match model output with workplace/school location model size terms, while the last method matches model output with actual employment/enrollmment data. **shadow_pricing.yaml Attributes** @@ -230,7 +230,7 @@ There are three shadow pricing methods in activitysim: ``ctramp``, ``daysim``, a - ``FAIL_THRESHOLD`` percentage of zones exceeding the PERCENT_TOLERANCE considered a failure - ``SHADOW_PRICE_METHOD`` [ctramp | daysim | simulation] - ``TOTAL_EMP`` total employment column name defined in the land use data -- ``TOTAL_ENORLLMENT``: total enrolmment colmns name defined in the land use data +- ``TOTAL_ENORLLMENT`` total enrollmment colmns name defined in the land use data - ``DAMPING_FACTOR`` On each iteration, ActivitySim will attempt to adjust the model to match desired size terms. The number is multiplied by adjustment factor to dampen or amplify the ActivitySim calculation. (only for CT-RAMP) - ``DAYSIM_ABSOLUTE_TOLERANCE`` From 2db5f59954701c18b7c794bf163facdab5bf6c6b Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 3 Oct 2022 12:08:58 -0700 Subject: [PATCH 12/27] fixing pandas future warning --- activitysim/abm/models/trip_purpose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/activitysim/abm/models/trip_purpose.py b/activitysim/abm/models/trip_purpose.py index fe3103135..bd2eb564b 100644 --- a/activitysim/abm/models/trip_purpose.py +++ b/activitysim/abm/models/trip_purpose.py @@ -71,7 +71,7 @@ def choose_intermediate_trip_purpose( # probs should sum to 1 across rows sum_probs = probs_spec[purpose_cols].sum(axis=1) - probs_spec.loc[:, purpose_cols] = probs_spec.loc[:, purpose_cols].div( + probs_spec[purpose_cols] = probs_spec[purpose_cols].div( sum_probs, axis=0 ) From b5768e32d2cd584076e068dfa790c14c7eff51b1 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 3 Oct 2022 12:36:52 -0700 Subject: [PATCH 13/27] blacken --- activitysim/abm/models/trip_purpose.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/activitysim/abm/models/trip_purpose.py b/activitysim/abm/models/trip_purpose.py index bd2eb564b..0e74382d8 100644 --- a/activitysim/abm/models/trip_purpose.py +++ b/activitysim/abm/models/trip_purpose.py @@ -71,9 +71,7 @@ def choose_intermediate_trip_purpose( # probs should sum to 1 across rows sum_probs = probs_spec[purpose_cols].sum(axis=1) - probs_spec[purpose_cols] = probs_spec[purpose_cols].div( - sum_probs, axis=0 - ) + probs_spec[purpose_cols] = probs_spec[purpose_cols].div(sum_probs, axis=0) # left join trips to probs (there may be multiple rows per trip for multiple depart ranges) choosers = pd.merge( From 30d67bccdcf59b026cf2420fae9953cf044c0b42 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 3 Oct 2022 13:29:43 -0700 Subject: [PATCH 14/27] bug in trying to access shadow price settings when not running shadow pricing --- activitysim/abm/models/location_choice.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index ef09617e9..439e25558 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -262,7 +262,9 @@ def aggregate_size_terms(dest_size_terms, network_los, model_settings): TAZ_size_terms[c] /= TAZ_size_terms["size_term"] # weighted average spc = shadow_pricing.load_shadow_price_calculator(model_settings) - if spc.shadow_settings["SHADOW_PRICE_METHOD"] == "simulation": + if spc.use_shadow_pricing and ( + spc.shadow_settings["SHADOW_PRICE_METHOD"] == "simulation" + ): # allow TAZs with at least one underassigned MAZ in them, therefore with a shadowprice larger than -999, to be selected again TAZ_size_terms["shadow_price_utility_adjustment"] = np.where( TAZ_size_terms["shadow_price_utility_adjustment"] > -999, 0, -999 From a20a396beddd86c982de40ecd11fc9c973749e15 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 3 Oct 2022 13:38:48 -0700 Subject: [PATCH 15/27] limiting pandas version --- conda-environments/github-actions-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-environments/github-actions-tests.yml b/conda-environments/github-actions-tests.yml index 01108bdc4..e626fbf7b 100644 --- a/conda-environments/github-actions-tests.yml +++ b/conda-environments/github-actions-tests.yml @@ -13,7 +13,7 @@ dependencies: - numpy >= 1.16.1,<=1.21 - openmatrix >= 0.3.4.1 - orca >= 1.6 -- pandas >= 1.1.0 +- pandas >= 1.1.0,<1.5 - psutil >= 4.1 - pyarrow >= 2.0 - pypyr >= 5.3 From 4d58a858cc16253a38117a4224e933c19a9ac24c Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 3 Oct 2022 14:41:27 -0700 Subject: [PATCH 16/27] always updating choices --- activitysim/abm/tables/shadow_pricing.py | 29 ++++++++++++------------ 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index c5fd0efb1..e2a9c1c76 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -431,23 +431,22 @@ def set_choices(self, choices, segment_ids): self.modeled_size = self.synchronize_modeled_size(modeled_size) # need to also store individual choices if simulation approach - if self.shadow_price_method == "simulation": - choice_merged = pd.merge( - self.shared_sp_choice_df, - choices, - left_index=True, - right_index=True, - how="left", - suffixes=("_x", "_y"), - ) + choice_merged = pd.merge( + self.shared_sp_choice_df, + choices, + left_index=True, + right_index=True, + how="left", + suffixes=("_x", "_y"), + ) - choice_merged["choice_y"] = choice_merged["choice_y"].fillna(0) - choice_merged["choice"] = ( - choice_merged["choice_x"] + choice_merged["choice_y"] - ) - choice_merged = choice_merged.drop(columns=["choice_x", "choice_y"]) + choice_merged["choice_y"] = choice_merged["choice_y"].fillna(0) + choice_merged["choice"] = ( + choice_merged["choice_x"] + choice_merged["choice_y"] + ) + choice_merged = choice_merged.drop(columns=["choice_x", "choice_y"]) - self.choices_synced = self.synchronize_choices(choice_merged) + self.choices_synced = self.synchronize_choices(choice_merged) def check_fit(self, iteration): """ From 7c706f71d5ba66b39c216ab7a598829fb4bbbd31 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 3 Oct 2022 16:28:25 -0700 Subject: [PATCH 17/27] testing removal of lognormal for hh vot --- .../examples/prototype_mtc/configs/annotate_households.csv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/activitysim/examples/prototype_mtc/configs/annotate_households.csv b/activitysim/examples/prototype_mtc/configs/annotate_households.csv index ac29e7472..3c4c19700 100644 --- a/activitysim/examples/prototype_mtc/configs/annotate_households.csv +++ b/activitysim/examples/prototype_mtc/configs/annotate_households.csv @@ -10,7 +10,8 @@ income_segment,income_segment,"pd.cut(income_in_thousands, bins=[-np.inf, 30, 60 ,_MU,setting('distributed_vot_mu') ,_SIGMA,setting('distributed_vot_sigma') median_value_of_time,median_value_of_time,"income_segment.map({k: v for k, v in setting('household_median_value_of_time').items()})" -hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)" +#hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)" +TEST hh_value_of_time,hh_value_of_time,1 #,, #num_workers was renamed in import,, #,num_workers,households.workers From e2e0d9d2caac5a7a57c897b1997d0d5f84b57691 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Mon, 3 Oct 2022 16:49:16 -0700 Subject: [PATCH 18/27] putting hh vot back in --- .../examples/prototype_mtc/configs/annotate_households.csv | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/activitysim/examples/prototype_mtc/configs/annotate_households.csv b/activitysim/examples/prototype_mtc/configs/annotate_households.csv index 3c4c19700..ac29e7472 100644 --- a/activitysim/examples/prototype_mtc/configs/annotate_households.csv +++ b/activitysim/examples/prototype_mtc/configs/annotate_households.csv @@ -10,8 +10,7 @@ income_segment,income_segment,"pd.cut(income_in_thousands, bins=[-np.inf, 30, 60 ,_MU,setting('distributed_vot_mu') ,_SIGMA,setting('distributed_vot_sigma') median_value_of_time,median_value_of_time,"income_segment.map({k: v for k, v in setting('household_median_value_of_time').items()})" -#hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)" -TEST hh_value_of_time,hh_value_of_time,1 +hh_value_of_time,hh_value_of_time,"rng.lognormal_for_df(df, mu=np.log(median_value_of_time * _MU), sigma=_SIGMA).clip(_MIN_VOT, _MAX_VOT)" #,, #num_workers was renamed in import,, #,num_workers,households.workers From 95d7fd7e9b29e5acc91a11f37f80839664f01522 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Tue, 4 Oct 2022 11:04:24 -0700 Subject: [PATCH 19/27] updating to match sharrow test versions --- conda-environments/github-actions-tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/conda-environments/github-actions-tests.yml b/conda-environments/github-actions-tests.yml index e626fbf7b..552bb74ec 100644 --- a/conda-environments/github-actions-tests.yml +++ b/conda-environments/github-actions-tests.yml @@ -1,4 +1,7 @@ # Environment for testing in GitHub Actions +# This environment contains a minimal set of dependencies needed to run most tests. +# It does not install ActivitySim itself (which is done by the test scripts) and +# is not meant for use outside the CI tools. name: asim-test channels: - conda-forge @@ -9,7 +12,7 @@ dependencies: - cytoolz >= 0.8.1 - isort - nbmake -- numba >= 0.55.2 +- numba = 0.55.2 # see https://github.com/conda-forge/numba-feedstock/pull/104 - numpy >= 1.16.1,<=1.21 - openmatrix >= 0.3.4.1 - orca >= 1.6 @@ -23,7 +26,7 @@ dependencies: - pytest-regressions - pyyaml >= 5.1 - requests >= 2.7 -- sharrow >= 2.2.4 +- sharrow >= 2.3.2 - simwrapper > 1.7 - xarray >= 0.21 -- zarr +- zarr \ No newline at end of file From 1ce6d36ea5647232466cb4ba7cbc90645eb47800 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 7 Oct 2022 15:12:16 -0700 Subject: [PATCH 20/27] raw person table for buffer instead of injectable --- activitysim/abm/tables/shadow_pricing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index e2a9c1c76..e9c845352 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -11,6 +11,7 @@ from activitysim.abm.tables.size_terms import tour_destination_size_terms from activitysim.core import config, inject, tracing, util +from activitysim.core.input import read_input_table logger = logging.getLogger(__name__) @@ -940,7 +941,7 @@ def buffers_for_shadow_pricing_choice(shadow_pricing_choice_info): data_buffers[block_key + "_choice"] = shared_data_buffer - persons = inject.get_table("persons").to_frame() + persons = read_input_table("persons") sp_choice_df = persons.reset_index()["person_id"].to_frame() # declare a shared Array with data from sp_choice_df @@ -1291,8 +1292,7 @@ def get_shadow_pricing_choice_info(): block_shapes: dict {: } """ - persons = inject.get_table("persons") - # size_terms = inject.get_injectable("size_terms") + persons = read_input_table("persons") shadow_settings = config.read_model_settings("shadow_pricing.yaml") @@ -1302,8 +1302,8 @@ def get_shadow_pricing_choice_info(): blocks = OrderedDict() for model_selector in shadow_pricing_models: + # each person will have a work or school location choice sp_rows = len(persons) - # sp_cols = len(size_terms[size_terms.model_selector == model_selector]) # extra tally column for TALLY_CHECKIN and TALLY_CHECKOUT semaphores blocks[block_name(model_selector)] = (sp_rows, 2) From 7167b65bf46009eec6e9563911edfe8d906420ae Mon Sep 17 00:00:00 2001 From: David Hensle Date: Tue, 25 Oct 2022 15:13:17 -0700 Subject: [PATCH 21/27] adding segmentation, output by iteration, and external worker removal --- activitysim/abm/tables/shadow_pricing.py | 318 ++++++++++++------ .../configs/annotate_landuse.csv | 12 +- .../configs/destination_choice_size_terms.csv | 28 ++ .../configs/settings.yaml | 5 +- .../configs/shadow_pricing.yaml | 27 +- 5 files changed, 285 insertions(+), 105 deletions(-) create mode 100644 activitysim/examples/prototype_mtc_extended/configs/destination_choice_size_terms.csv diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index e9c845352..e0ed43da1 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -54,6 +54,12 @@ TALLY_CHECKIN = (0, -1) TALLY_CHECKOUT = (1, -1) +default_segment_to_name_dict = { + # model_selector : persons_segment_name + "school": "school_segment", + "workplace": "income_segment", +} + def size_table_name(model_selector): """ @@ -168,6 +174,8 @@ def __init__( self.shadow_prices = None self.shadow_price_method = self.shadow_settings["SHADOW_PRICE_METHOD"] assert self.shadow_price_method in ["daysim", "ctramp", "simulation"] + # ignore convergence criteria for zones smaller than target_threshold + self.target_threshold = self.shadow_settings["TARGET_THRESHOLD"] if self.shadow_settings["LOAD_SAVED_SHADOW_PRICES"]: # read_saved_shadow_prices logs error and returns None if file not found @@ -197,6 +205,7 @@ def __init__( self.num_fail = pd.DataFrame(index=self.desired_size.columns) self.max_abs_diff = pd.DataFrame(index=self.desired_size.columns) self.max_rel_diff = pd.DataFrame(index=self.desired_size.columns) + self.choices_by_iteration = pd.DataFrame() if ( self.use_shadow_pricing @@ -204,17 +213,41 @@ def __init__( ): assert self.model_selector in ["workplace", "school"] + self.sampled_persons = pd.DataFrame() + self.target = {} + land_use = inject.get_table("land_use").to_frame() if self.model_selector == "workplace": - self.total_emp = self.shadow_settings["TOTAL_EMP"] - land_use = inject.get_table("land_use").to_frame() - self.target = land_use[self.total_emp] + employment_targets = self.shadow_settings[ + "workplace_segmentation_targets" + ] + assert ( + employment_targets is not None + ), "Need to supply workplace_segmentation_targets in shadow_pricing.yaml" + + for segment, target in employment_targets.items(): + assert ( + segment in self.shadow_prices.columns + ), f"{segment} is not in {self.shadow_prices.columns}" + assert ( + target in land_use.columns + ), f"{target} is not in {land_use.columns}" + self.target[segment] = land_use[target] elif self.model_selector == "school": - total_enr = self.shadow_settings["TOTAL_ENROLLMENT"] - land_use = inject.get_table("land_use").to_frame() - self.target = land_use[total_enr] - self.zonal_sample_rate = None + school_targets = self.shadow_settings["school_segmentation_targets"] + assert ( + school_targets is not None + ), "Need to supply school_segmentation_targets in shadow_pricing.yaml" + + for segment, target in school_targets.items(): + assert ( + segment in self.shadow_prices.columns + ), f"{segment} is not in {self.shadow_prices.columns}" + assert ( + target in land_use.columns + ), f"{target} is not in landuse columns: {land_use.columns}" + self.target[segment] = land_use[target] def read_saved_shadow_prices(self, model_settings): """ @@ -481,6 +514,10 @@ def check_fit(self, iteration): percent_tolerance = self.shadow_settings["PERCENT_TOLERANCE"] # max percentage of zones allowed to fail fail_threshold = self.shadow_settings["FAIL_THRESHOLD"] + # option to write out choices by iteration for each person to trace folder + write_choices = self.shadow_settings.get("WRITE_ITERATION_CHOICES", False) + if write_choices: + self.choices_by_iteration[iteration] = self.choices_synced if self.shadow_settings["SHADOW_PRICE_METHOD"] != "simulation": @@ -510,44 +547,45 @@ def check_fit(self, iteration): converged = total_fails <= max_fail - logger.info( - "check_fit %s iteration: %s converged: %s max_fail: %s total_fails: %s" - % (self.model_selector, iteration, converged, max_fail, total_fails) - ) - - # - convergence stats - if converged or iteration == self.max_iterations: - logger.info("\nshadow_pricing max_abs_diff\n%s" % self.max_abs_diff) - logger.info("\nshadow_pricing max_rel_diff\n%s" % self.max_rel_diff) - logger.info("\nshadow_pricing num_fail\n%s" % self.num_fail) - else: - # ignore convergence criteria for zones smaller than target_threshold - self.target_threshold = self.shadow_settings["TARGET_THRESHOLD"] - - modeled_size = self.modeled_size - desired_size = self.target - - desired_share = self.target / self.target.sum() - modeled_share = ( - self.modeled_size.sum(axis=1) / self.modeled_size.sum().sum() - ) - - self.rel_diff = desired_share / modeled_share + rel_diff_df = pd.DataFrame(index=self.shadow_prices.index) + abs_diff_df = pd.DataFrame(index=self.shadow_prices.index) + # checking each segment + for segment in self.segment_ids: + desired_size = self.target[segment] + modeled_size = self.modeled_size[segment] + + # loop over other segments and add to modeled share if they have the same target + for other_segment in self.segment_ids: + if (segment != other_segment) & ( + self.target[segment].equals(self.target[other_segment]) + ): + modeled_size = modeled_size + self.modeled_size[other_segment] + + # want to match distribution, not absolute numbers so share is computed + desired_share = desired_size / desired_size.sum() + modeled_share = modeled_size / modeled_size.sum() + + abs_diff_df[segment] = (desired_size - modeled_size).abs() + + rel_diff = desired_share / modeled_share + rel_diff = np.where( + # is the desired size below the threshold? + (desired_size <= self.target_threshold) + # is the difference within the tolerance? + | (np.abs(1 - rel_diff) < (percent_tolerance / 100.0)), + 0, + rel_diff, + ) + rel_diff_df[segment] = rel_diff - # ignore zones where desired_size < threshold - self.rel_diff.where(desired_size >= self.target_threshold, 0, inplace=True) + # relative difference is set to max across segments + self.rel_diff = rel_diff_df.max(axis=1) + abs_diff = abs_diff_df.max(axis=1) - # ignore zones where rel_diff is within percent_tolerance - self.rel_diff.where( - (self.rel_diff > 1 + (percent_tolerance / 100.0)) - | (self.rel_diff < 1 - (percent_tolerance / 100.0)), - 0, - inplace=True, - ) self.num_fail["iter%s" % iteration] = (self.rel_diff > 0).sum() - # self.max_abs_diff["iter%s" % iteration] = abs_diff.max() - # self.max_rel_diff["iter%s" % iteration] = rel_diff.max() + self.max_abs_diff["iter%s" % iteration] = abs_diff.max() + self.max_rel_diff["iter%s" % iteration] = rel_diff.max() total_fails = (self.rel_diff > 0).values.sum() @@ -555,19 +593,26 @@ def check_fit(self, iteration): max_fail = (fail_threshold / 100.0) * util.iprod(desired_size.shape) converged = (total_fails <= np.ceil(max_fail)) | ( - len(self.choices_synced) == 0 + (iteration > 1) & (len(self.sampled_persons) == 0) ) - logger.info( - "check_fit %s iteration: %s converged: %s max_fail: %s total_fails: %s" - % (self.model_selector, iteration, converged, max_fail, total_fails) - ) + logger.info( + "check_fit %s iteration: %s converged: %s max_fail: %s total_fails: %s" + % (self.model_selector, iteration, converged, max_fail, total_fails) + ) - # - convergence stats - if converged or iteration == self.max_iterations: - logger.info("\nshadow_pricing max_abs_diff\n%s" % self.max_abs_diff) - logger.info("\nshadow_pricing max_rel_diff\n%s" % self.max_rel_diff) - logger.info("\nshadow_pricing num_fail\n%s" % self.num_fail) + # - convergence stats + if converged or iteration == self.max_iterations: + logger.info("\nshadow_pricing max_abs_diff\n%s" % self.max_abs_diff) + logger.info("\nshadow_pricing max_rel_diff\n%s" % self.max_rel_diff) + logger.info("\nshadow_pricing num_fail\n%s" % self.num_fail) + + if write_choices: + tracing.write_csv( + self.choices_by_iteration, + "%s_choices_by_shadow_price_iteration" % self.model_selector, + transpose=False, + ) return converged @@ -692,61 +737,144 @@ def update_shadow_prices(self): shadow_price_j = -999 resimulate n workers from zone j, with n = int(workers_j-emp_j/sum(emp_j*workers_j)) """ + percent_tolerance = self.shadow_settings["PERCENT_TOLERANCE"] + sampled_persons = pd.DataFrame() + persons_merged = inject.get_table("persons_merged").to_frame() - desired_share = self.target / self.target.sum() - modeled_share = ( - self.modeled_size.sum(axis=1) / self.modeled_size.sum().sum() + # need to join the segment to the choices to sample correct persons + segment_to_name_dict = self.shadow_settings.get( + "", default_segment_to_name_dict + ) + segment_name = segment_to_name_dict[self.model_selector] + choices_synced = ( + self.choices_synced.to_frame() + .merge( + persons_merged[segment_name], + how="left", + left_index=True, + right_index=True, + ) + .rename(columns={segment_name: "segment"}) ) - percent_tolerance = self.shadow_settings["PERCENT_TOLERANCE"] - sprice = desired_share / modeled_share - sprice.fillna(0, inplace=True) - sprice.replace([np.inf, -np.inf], 0, inplace=True) - - adjustment_ = np.where(sprice <= 1 + percent_tolerance / 100, -999, 0) - adjustment = pd.DataFrame(index=self.shadow_prices.index) - - for seg_id in self.shadow_prices.columns: - adjustment[seg_id] = adjustment_ - - new_shadow_prices = adjustment - self.zonal_sample_rate = 1 - sprice - overpredicted_zones = new_shadow_prices[ - new_shadow_prices.iloc[:, 0] == -999 - ].index - zones_outside_tol = self.zonal_sample_rate[ - self.zonal_sample_rate > percent_tolerance / 100 - ].index - small_zones = self.target[self.target <= self.target_threshold].index - - choices = self.choices_synced[ - (self.choices_synced.choice.isin(overpredicted_zones)) - & (self.choices_synced.choice.isin(zones_outside_tol)) - & ~(self.choices_synced.choice.isin(small_zones)) - ] + for segment in self.segment_ids: + desired_size = self.target[segment] + modeled_size = self.modeled_size[segment] + + # loop over other segments and add to modeled share if they have the same target + for other_segment in self.segment_ids: + if (segment != other_segment) & ( + self.target[segment].equals(self.target[other_segment]) + ): + modeled_size = modeled_size + self.modeled_size[other_segment] + + # want to match distribution, not absolute numbers so share is computed + desired_share = desired_size / desired_size.sum() + modeled_share = modeled_size / modeled_size.sum() - choices_index = choices.index.name - choices = choices.reset_index() + sprice = desired_share / modeled_share + sprice.fillna(0, inplace=True) + sprice.replace([np.inf, -np.inf], 0, inplace=True) - # handling unlikely cases where there are no more overassigned zones, but a few underassigned zones remain - if len(choices) > 0: - self.sampled_persons = ( - choices.groupby("choice") - .apply( - lambda x: x.sample( - frac=self.zonal_sample_rate.loc[x.name], random_state=1 + # shadow prices are set to -999 if overassigned or 0 if the zone still has room for this segment + self.shadow_prices[segment] = np.where( + (sprice <= 1 + percent_tolerance / 100), -999, 0 + ) + + zonal_sample_rate = 1 - sprice + overpredicted_zones = self.shadow_prices[ + self.shadow_prices[segment] == -999 + ].index + zones_outside_tol = zonal_sample_rate[ + zonal_sample_rate > percent_tolerance / 100 + ].index + small_zones = desired_size[desired_size <= self.target_threshold].index + + choices = choices_synced[ + (choices_synced["choice"].isin(overpredicted_zones)) + & (choices_synced["choice"].isin(zones_outside_tol)) + & ~(choices_synced["choice"].isin(small_zones)) + # sampling only from people in this segment + & (choices_synced["segment"] == self.segment_ids[segment]) + ]["choice"] + + choices_index = choices.index.name + choices = choices.reset_index() + + # handling unlikely cases where there are no more overassigned zones, but a few underassigned zones remain + if len(choices) > 0: + current_sample = ( + choices.groupby("choice") + .apply( + # FIXME is this sample stable? + lambda x: x.sample( + frac=zonal_sample_rate.loc[x.name], random_state=1 + ) ) + .reset_index(drop=True) + .set_index(choices_index) ) - .reset_index(drop=True) - .set_index(choices_index) - ) - else: - self.sampled_persons = pd.DataFrame() + if len(sampled_persons) == 0: + sampled_persons = current_sample + else: + sampled_persons = pd.concat([sampled_persons, current_sample]) + + self.sampled_persons = sampled_persons + + # desired_share = self.target / self.target.sum() + # modeled_share = ( + # self.modeled_size.sum(axis=1) / self.modeled_size.sum().sum() + # ) + # percent_tolerance = self.shadow_settings["PERCENT_TOLERANCE"] + + # sprice = desired_share / modeled_share + # sprice.fillna(0, inplace=True) + # sprice.replace([np.inf, -np.inf], 0, inplace=True) + + # adjustment_ = np.where(sprice <= 1 + percent_tolerance / 100, -999, 0) + # adjustment = pd.DataFrame(index=self.shadow_prices.index) + + # for seg_id in self.shadow_prices.columns: + # adjustment[seg_id] = adjustment_ + + # new_shadow_prices = adjustment + # self.zonal_sample_rate = 1 - sprice + # overpredicted_zones = new_shadow_prices[ + # new_shadow_prices.iloc[:, 0] == -999 + # ].index + # zones_outside_tol = self.zonal_sample_rate[ + # self.zonal_sample_rate > percent_tolerance / 100 + # ].index + # small_zones = self.target[self.target <= self.target_threshold].index + + # choices = self.choices_synced[ + # (self.choices_synced.choice.isin(overpredicted_zones)) + # & (self.choices_synced.choice.isin(zones_outside_tol)) + # & ~(self.choices_synced.choice.isin(small_zones)) + # ] + + # choices_index = choices.index.name + # choices = choices.reset_index() + + # # handling unlikely cases where there are no more overassigned zones, but a few underassigned zones remain + # if len(choices) > 0: + # self.sampled_persons = ( + # choices.groupby("choice") + # .apply( + # lambda x: x.sample( + # frac=self.zonal_sample_rate.loc[x.name], random_state=1 + # ) + # ) + # .reset_index(drop=True) + # .set_index(choices_index) + # ) + # else: + # self.sampled_persons = pd.DataFrame() else: raise RuntimeError("unknown SHADOW_PRICE_METHOD %s" % shadow_price_method) - self.shadow_prices = new_shadow_prices + # self.shadow_prices = new_shadow_prices def dest_size_terms(self, segment): diff --git a/activitysim/examples/prototype_mtc_extended/configs/annotate_landuse.csv b/activitysim/examples/prototype_mtc_extended/configs/annotate_landuse.csv index 9b1dd5eb6..a434db9db 100644 --- a/activitysim/examples/prototype_mtc_extended/configs/annotate_landuse.csv +++ b/activitysim/examples/prototype_mtc_extended/configs/annotate_landuse.csv @@ -4,4 +4,14 @@ household_density,household_density,land_use.TOTHH / (land_use.RESACRE + land_us employment_density,employment_density,land_use.TOTEMP / (land_use.RESACRE + land_use.CIACRE) density_index,density_index,(household_density *employment_density) / (household_density + employment_density).clip(lower=1) ,is_cbd,land_use.area_type == 1 -total enrollment for simulation based shadow pricing,TOTENR,land_use.AGE0519 + land_use.HSENROLL + land_use.COLLFTE + land_use.COLLPTE \ No newline at end of file +# additions put in place for simulation shadow pricing approach,, +total university enrollment,TOTENR_univ,land_use.COLLFTE + land_use.COLLPTE +# example external worker implementation,, +Example with 10 percent external workers across all zones,ext_work_share,0.1 +scaling employment fields,RETEMPN_scaled,land_use.RETEMPN * (1 - ext_work_share) +,FPSEMPN_scaled,land_use.FPSEMPN * (1 - ext_work_share) +,HEREMPN_scaled,land_use.HEREMPN * (1 - ext_work_share) +,OTHEMPN_scaled,land_use.OTHEMPN * (1 - ext_work_share) +,AGREMPN_scaled,land_use.AGREMPN * (1 - ext_work_share) +,MWTEMPN_scaled,land_use.MWTEMPN * (1 - ext_work_share) +,TOTEMP_scaled,land_use.TOTEMP * (1 - ext_work_share) diff --git a/activitysim/examples/prototype_mtc_extended/configs/destination_choice_size_terms.csv b/activitysim/examples/prototype_mtc_extended/configs/destination_choice_size_terms.csv new file mode 100644 index 000000000..f07de5d48 --- /dev/null +++ b/activitysim/examples/prototype_mtc_extended/configs/destination_choice_size_terms.csv @@ -0,0 +1,28 @@ +model_selector,segment,TOTHH,RETEMPN,FPSEMPN,HEREMPN,OTHEMPN,AGREMPN,MWTEMPN,AGE0519,HSENROLL,COLLFTE,COLLPTE,RETEMPN_scaled,FPSEMPN_scaled,HEREMPN_scaled,OTHEMPN_scaled,AGREMPN_scaled,MWTEMPN_scaled +workplace,work_low,0,0,0,0,0,0,0,0,0,0,0,0.129,0.193,0.383,0.12,0.01,0.164 +workplace,work_med,0,0,0,0,0,0,0,0,0,0,0,0.12,0.197,0.325,0.139,0.008,0.21 +workplace,work_high,0,0,0,0,0,0,0,0,0,0,0,0.11,0.207,0.284,0.154,0.006,0.239 +workplace,work_veryhigh,0,0,0,0,0,0,0,0,0,0,0,0.093,0.27,0.241,0.146,0.004,0.246 +school,university,0,0,0,0,0,0,0,0,0,0.592,0.408,0,0,0,0,0,0 +school,gradeschool,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0 +school,highschool,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0 +non_mandatory,escort,0,0.225,0,0.144,0,0,0,0.465,0.166,0,0,0,0,0,0,0,0 +#non_mandatory,escort_kids,0,0.225,0,0.144,0,0,0,0.465,0.166,0,0,0,0,0,0,0,0 +#non_mandatory,escort_nokids,0,0.225,0,0.144,0,0,0,0.465,0.166,0,0,0,0,0,0,0,0 +non_mandatory,shopping,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +non_mandatory,eatout,0,0.742,0,0.258,0,0,0,0,0,0,0,0,0,0,0,0,0 +non_mandatory,othmaint,0,0.482,0,0.518,0,0,0,0,0,0,0,0,0,0,0,0,0 +non_mandatory,social,0,0.522,0,0.478,0,0,0,0,0,0,0,0,0,0,0,0,0 +non_mandatory,othdiscr,0.252,0.212,0,0.272,0.165,0,0,0,0.098,0,0,0,0,0,0,0,0 +atwork,atwork,0,0.742,0,0.258,0,0,0,0,0,0,0,0,0,0,0,0,0 +trip,work,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0 +trip,escort,0.001,0.225,0,0.144,0,0,0,0.464,0.166,0,0,0,0,0,0,0,0 +trip,shopping,0.001,0.999,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +trip,eatout,0,0.742,0,0.258,0,0,0,0,0,0,0,0,0,0,0,0,0 +trip,othmaint,0.001,0.481,0,0.518,0,0,0,0,0,0,0,0,0,0,0,0,0 +trip,social,0.001,0.521,0,0.478,0,0,0,0,0,0,0,0,0,0,0,0,0 +trip,othdiscr,0.252,0.212,0,0.272,0.165,0,0,0,0.098,0,0,0,0,0,0,0,0 +trip,univ,0.001,0,0,0,0,0,0,0,0,0.592,0.408,0,0,0,0,0,0 +# not needed as school is not chosen as an intermediate trip destination,,,,,,,,,,,,,,,,,, +#trip,gradeschool,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0 +#trip,highschool,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0 diff --git a/activitysim/examples/prototype_mtc_extended/configs/settings.yaml b/activitysim/examples/prototype_mtc_extended/configs/settings.yaml index b1faa5c25..b7bc1ff03 100644 --- a/activitysim/examples/prototype_mtc_extended/configs/settings.yaml +++ b/activitysim/examples/prototype_mtc_extended/configs/settings.yaml @@ -102,7 +102,7 @@ check_for_variability: False # turn shadow_pricing on and off for all models (e.g. school and work) # shadow pricing is deprecated for less than full samples # see shadow_pricing.yaml for additional settings -use_shadow_pricing: False +use_shadow_pricing: True # turn writing of sample_tables on and off for all models # (if True, tables will be written if DEST_CHOICE_SAMPLE_TABLE_NAME is specified in individual model settings) @@ -155,7 +155,7 @@ keep_mem_logs: True # trace household id; comment out or leave empty for no trace # households with all tour types # [ 728370 1234067 1402924 1594625 1595333 1747572 1896849 1931818 2222690 2344951 2677154] -trace_hh_id: 982875 +trace_hh_id: # trace origin, destination in accessibility calculation; comment out or leave empty for no trace # trace_od: [5, 11] @@ -165,7 +165,6 @@ trace_od: # to resume after last successful checkpoint, specify resume_after: _ #resume_after: trip_destination -resume_after: checkpoints: True # if checkpoints is False, no intermediate checkpoints will be written before the end of run diff --git a/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml b/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml index e314e918e..f2fb2cbbd 100644 --- a/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml +++ b/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml @@ -2,10 +2,28 @@ shadow_pricing_models: school: school_location workplace: workplace_location +# apply different targets for each segment specified in destination_size_terms.csv +school_segmentation_targets: + # format is segment: land_use_column + university: TOTENR_univ + highschool: HSENROLL + gradeschool: AGE0519 + +# if target names are the same, they will be combined together +workplace_segmentation_targets: + # using total employment scaled to remove external workers. see annotate_landuse.csv + work_low: TOTEMP_scaled + work_med: TOTEMP_scaled + work_high: TOTEMP_scaled + work_veryhigh: TOTEMP_scaled + # global switch to enable/disable loading of saved shadow prices # (ignored if global use_shadow_pricing switch is False) LOAD_SAVED_SHADOW_PRICES: False +# write out choices by iteration to trace folder +WRITE_ITERATION_CHOICES: True + # number of shadow price iterations for cold start MAX_ITERATIONS: 10 @@ -18,9 +36,6 @@ MAX_ITERATIONS_SAVED: 1 SHADOW_PRICE_METHOD: simulation # --- simulation method settings -# total employment/enrollment fields in landuse -TOTAL_EMP: TOTEMP -TOTAL_ENROLLMENT: TOTENR # ignore criteria for zones smaller than size_threshold SIZE_THRESHOLD: 10 # ignore criteria for zones smaller than target_threshold (total employmnet or enrollment) @@ -31,10 +46,10 @@ PERCENT_TOLERANCE: 5 FAIL_THRESHOLD: 1 # --- ctramp method settings -# DAMPING_FACTOR: 1 +DAMPING_FACTOR: 1 # --- daysim method settings # FIXME should these be the same as PERCENT_TOLERANCE and FAIL_THRESHOLD above? -# DAYSIM_ABSOLUTE_TOLERANCE: 50 -# DAYSIM_PERCENT_TOLERANCE: 10 +DAYSIM_ABSOLUTE_TOLERANCE: 50 +DAYSIM_PERCENT_TOLERANCE: 10 From bce64bd0b8f2f2401a56bed2853190debc460226 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Thu, 27 Oct 2022 14:56:46 -0700 Subject: [PATCH 22/27] formatting & documentation --- activitysim/abm/tables/shadow_pricing.py | 66 ++----------- .../configs/shadow_pricing.yaml | 29 +++--- docs/models.rst | 93 +++++++++++++------ 3 files changed, 85 insertions(+), 103 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index e0ed43da1..8a3310819 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -746,16 +746,12 @@ def update_shadow_prices(self): "", default_segment_to_name_dict ) segment_name = segment_to_name_dict[self.model_selector] - choices_synced = ( - self.choices_synced.to_frame() - .merge( - persons_merged[segment_name], - how="left", - left_index=True, - right_index=True, - ) - .rename(columns={segment_name: "segment"}) - ) + choices_synced = self.choices_synced.merge( + persons_merged[segment_name], + how="left", + left_index=True, + right_index=True, + ).rename(columns={segment_name: "segment"}) for segment in self.segment_ids: desired_size = self.target[segment] @@ -821,56 +817,6 @@ def update_shadow_prices(self): self.sampled_persons = sampled_persons - # desired_share = self.target / self.target.sum() - # modeled_share = ( - # self.modeled_size.sum(axis=1) / self.modeled_size.sum().sum() - # ) - # percent_tolerance = self.shadow_settings["PERCENT_TOLERANCE"] - - # sprice = desired_share / modeled_share - # sprice.fillna(0, inplace=True) - # sprice.replace([np.inf, -np.inf], 0, inplace=True) - - # adjustment_ = np.where(sprice <= 1 + percent_tolerance / 100, -999, 0) - # adjustment = pd.DataFrame(index=self.shadow_prices.index) - - # for seg_id in self.shadow_prices.columns: - # adjustment[seg_id] = adjustment_ - - # new_shadow_prices = adjustment - # self.zonal_sample_rate = 1 - sprice - # overpredicted_zones = new_shadow_prices[ - # new_shadow_prices.iloc[:, 0] == -999 - # ].index - # zones_outside_tol = self.zonal_sample_rate[ - # self.zonal_sample_rate > percent_tolerance / 100 - # ].index - # small_zones = self.target[self.target <= self.target_threshold].index - - # choices = self.choices_synced[ - # (self.choices_synced.choice.isin(overpredicted_zones)) - # & (self.choices_synced.choice.isin(zones_outside_tol)) - # & ~(self.choices_synced.choice.isin(small_zones)) - # ] - - # choices_index = choices.index.name - # choices = choices.reset_index() - - # # handling unlikely cases where there are no more overassigned zones, but a few underassigned zones remain - # if len(choices) > 0: - # self.sampled_persons = ( - # choices.groupby("choice") - # .apply( - # lambda x: x.sample( - # frac=self.zonal_sample_rate.loc[x.name], random_state=1 - # ) - # ) - # .reset_index(drop=True) - # .set_index(choices_index) - # ) - # else: - # self.sampled_persons = pd.DataFrame() - else: raise RuntimeError("unknown SHADOW_PRICE_METHOD %s" % shadow_price_method) diff --git a/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml b/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml index f2fb2cbbd..593a36478 100644 --- a/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml +++ b/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml @@ -2,21 +2,6 @@ shadow_pricing_models: school: school_location workplace: workplace_location -# apply different targets for each segment specified in destination_size_terms.csv -school_segmentation_targets: - # format is segment: land_use_column - university: TOTENR_univ - highschool: HSENROLL - gradeschool: AGE0519 - -# if target names are the same, they will be combined together -workplace_segmentation_targets: - # using total employment scaled to remove external workers. see annotate_landuse.csv - work_low: TOTEMP_scaled - work_med: TOTEMP_scaled - work_high: TOTEMP_scaled - work_veryhigh: TOTEMP_scaled - # global switch to enable/disable loading of saved shadow prices # (ignored if global use_shadow_pricing switch is False) LOAD_SAVED_SHADOW_PRICES: False @@ -44,6 +29,20 @@ TARGET_THRESHOLD: 20 PERCENT_TOLERANCE: 5 # max percentage of zones allowed to fail FAIL_THRESHOLD: 1 +# apply different targets for each segment specified in destination_size_terms.csv +school_segmentation_targets: + # format is segment: land_use_column + university: TOTENR_univ + highschool: HSENROLL + gradeschool: AGE0519 + +# if target names are the same, they will be combined together +workplace_segmentation_targets: + # using total employment scaled to remove external workers. see annotate_landuse.csv + work_low: TOTEMP_scaled + work_med: TOTEMP_scaled + work_high: TOTEMP_scaled + work_veryhigh: TOTEMP_scaled # --- ctramp method settings DAMPING_FACTOR: 1 diff --git a/docs/models.rst b/docs/models.rst index 200504694..06468a96c 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -197,44 +197,81 @@ The shadow pricing calculator used by work and school location choice. **Turning on and saving shadow prices** -Shadow pricing is activated by setting the ``use_shadow_pricing`` to True in the settings.yaml file. Once this setting has -been activated, ActivitySim will search for shadow pricing configuration in the shadow_pricing.yaml file. When shadow pricing is -activated, the shadow pricing outputs will be exported by the tracing engine. As a result, the shadow pricing output files will -be prepended with ``trace`` followed by the iteration number the results represent. For example, the shadow pricing outputs -for iteration 3 of the school location model will be called ``trace.shadow_price_school_shadow_prices_3.csv``. +Shadow pricing is activated by setting the ``use_shadow_pricing`` to True in the settings.yaml file. +Once this setting has been activated, ActivitySim will search for shadow pricing configuration in +the shadow_pricing.yaml file. When shadow pricing is activated, the shadow pricing outputs will be +exported by the tracing engine. As a result, the shadow pricing output files will be prepended with +``trace`` followed by the iteration number the results represent. For example, the shadow pricing +outputs for iteration 3 of the school location model will be called +``trace.shadow_price_school_shadow_prices_3.csv``. In total, ActivitySim generates three types of output files for each model with shadow pricing: -- ``trace.shadow_price__desired_size.csv`` - The size terms by zone that the ctramp and daysim methods are attempting to target. These equal the size term columns in the land use data multiplied by size term coefficients. - -- ``trace.shadow_price__modeled_size_.csv`` - These are the modeled size terms after the iteration of shadow pricing identified by the number. In other - words, these are the predicted choices by zone and segment for the model after the iteration completes. - -- ``trace.shadow_price__shadow_prices_.csv`` - The actual shadow price for each zone and segment after the of shadow pricing. This is the file that can be - used to warm start the shadow pricing mechanism in ActivitySim. - -There are three shadow pricing methods in activitysim: ``ctramp``, ``daysim``, and ``simulation``. The first two methods try to match model output with workplace/school location model size terms, while the last method matches model output with actual employment/enrollmment data. +- ``trace.shadow_price__desired_size.csv`` The size terms by zone that the ctramp and daysim + methods are attempting to target. These equal the size term columns in the land use data + multiplied by size term coefficients. + +- ``trace.shadow_price__modeled_size_.csv`` These are the modeled size terms after + the iteration of shadow pricing identified by the number. In other words, these are + the predicted choices by zone and segment for the model after the iteration completes. (Not + applicable for ``simulation`` option.) + +- ``trace.shadow_price__shadow_prices_.csv`` The actual shadow price for each zone + and segment after the of shadow pricing. This is the file that can be used to warm + start the shadow pricing mechanism in ActivitySim. (Not applicable for ``simulation`` option.) + +There are three shadow pricing methods in activitysim: ``ctramp``, ``daysim``, and ``simulation``. +The first two methods try to match model output with workplace/school location model size terms, +while the last method matches model output with actual employment/enrollmment data. + +The simulation approach operates the following steps. First, every worker / student will be +assigned without shadow prices applied. The modeled share and the target share for each zone are +compared. If the zone is overassigned, a sample of people from the over-assigned zones will be +selected for re-simulation. Shadow prices are set to -999 for the next iteration for overassigned +zones which removes the zone from the set of alternatives in the next iteration. The sampled people +will then be forced to choose from one of the under-assigned zones that still have the initial +shadow price of 0. (In this approach, the shadow price variable is really just a switch turning that +zone on or off for selection in the subsequent iterations. For this reason, warm-start functionality +for this approach is not applicable.) This process repeats until the overall convergence criteria +is met or the maximum number of allowed iterations is reached. + +Because the simulation approach only re-simulates workers / students who were over-assigned in the +previous iteration, run time is significantly less (~90%) than the CTRAMP or DaySim approaches which +re-simulate all workers and students at each iteration. **shadow_pricing.yaml Attributes** -- ``shadow_pricing_models`` List model_selectors and model_names of models that use shadow pricing. This list identifies which size_terms to preload which must be done in single process mode, so predicted_size tables can be scaled to population) -- ``LOAD_SAVED_SHADOW_PRICES`` global switch to enable/disable loading of saved shadow prices. From the above example, this would be trace.shadow_price__shadow_prices_.csv renamed and stored in the ``data_dir``. -- ``MAX_ITERATIONS`` If no loaded shadow prices, maximum number of times shadow pricing can be run on each model before proceeding to the next model. -- ``MAX_ITERATIONS_SAVED`` If loaded shadow prices, maximum number of times shadow pricing can be run. -- ``SIZE_THRESHOLD`` Ignore zones in failure calculation (ctramp or daysim method) with smaller size term value than size_threshold. -- ``TARGET_THRESHOLD`` Ignore zones in failure calculation (simulation method) with smaller employment/enrollment than target_threshold. +- ``shadow_pricing_models`` List model_selectors and model_names of models that use shadow pricing. + This list identifies which size_terms to preload which must be done in single process mode, so + predicted_size tables can be scaled to population +- ``LOAD_SAVED_SHADOW_PRICES`` global switch to enable/disable loading of saved shadow prices. From + the above example, this would be trace.shadow_price__shadow_prices_.csv renamed + and stored in the ``data_dir``. +- ``MAX_ITERATIONS`` If no loaded shadow prices, maximum number of times shadow pricing can be run + on each model before proceeding to the next model. +- ``MAX_ITERATIONS_SAVED`` If loaded shadow prices, maximum number of times shadow pricing can be + run. +- ``SIZE_THRESHOLD`` Ignore zones in failure calculation (ctramp or daysim method) with smaller size + term value than size_threshold. +- ``TARGET_THRESHOLD`` Ignore zones in failure calculation (simulation method) with smaller + employment/enrollment than target_threshold. - ``PERCENT_TOLERANCE`` Maximum percent difference between modeled and desired size terms - ``FAIL_THRESHOLD`` percentage of zones exceeding the PERCENT_TOLERANCE considered a failure - ``SHADOW_PRICE_METHOD`` [ctramp | daysim | simulation] -- ``TOTAL_EMP`` total employment column name defined in the land use data -- ``TOTAL_ENORLLMENT`` total enrollmment colmns name defined in the land use data -- ``DAMPING_FACTOR`` On each iteration, ActivitySim will attempt to adjust the model to match desired size terms. The number is multiplied by adjustment factor to dampen or amplify the ActivitySim calculation. (only for CT-RAMP) -- ``DAYSIM_ABSOLUTE_TOLERANCE`` +- ``workplace_segmentation_targets`` dict matching school segment to landuse employment column + target. Only used as part of simulation option. If mutiple segments list the same target column, + the segments will be added together for comparison. (Same with the school option below.) +- ``school_segmentation_targets`` dict matching school segment to landuse enrollment column target. + Only used as part of simulation option. +- ``DAMPING_FACTOR`` On each iteration, ActivitySim will attempt to adjust the model to match + desired size terms. The number is multiplied by adjustment factor to dampen or amplify the + ActivitySim calculation. (only for CTRAMP) +- ``DAYSIM_ABSOLUTE_TOLERANCE`` Absolute tolerance for DaySim option +- ``DAYSIM_PERCENT_TOLERANCE`` Relative tolerance for DaySim option +- ``WRITE_ITERATION_CHOICES`` [True | False ] Writes the choices of each person out to the trace + folder. Used for debugging or checking itration convergence. WARNING: every person is written for + each sub-process so the disc space can get large. -- ``DAYSIM_PERCENT_TOLERANCE`` .. automodule:: activitysim.abm.tables.shadow_pricing :members: From ebd2bc859416c7d40a89fbb3ada40f22a14a70c7 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 4 Nov 2022 16:36:58 -0700 Subject: [PATCH 23/27] ensuring TAZ is not selected if no available MAZ --- activitysim/abm/models/location_choice.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 439e25558..5abf8f26c 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -243,6 +243,13 @@ def aggregate_size_terms(dest_size_terms, network_los, model_settings): ) MAZ_size_terms[DEST_TAZ] = MAZ_size_terms.index.map(maz_to_taz) + MAZ_size_terms["avail_MAZ"] = np.where( + (MAZ_size_terms.size_term > 0) + & (MAZ_size_terms.shadow_price_utility_adjustment > -999), + 1, + 0, + ) + weighted_average_cols = [ "shadow_price_size_term_adjustment", "shadow_price_utility_adjustment", @@ -267,7 +274,10 @@ def aggregate_size_terms(dest_size_terms, network_los, model_settings): ): # allow TAZs with at least one underassigned MAZ in them, therefore with a shadowprice larger than -999, to be selected again TAZ_size_terms["shadow_price_utility_adjustment"] = np.where( - TAZ_size_terms["shadow_price_utility_adjustment"] > -999, 0, -999 + (TAZ_size_terms["shadow_price_utility_adjustment"] > -999) + & (TAZ_size_terms["avail_MAZ"] > 0), + 0, + -999, ) # now, negative size term means shadow price is -999. Setting size_term to 0 so the prob of that MAZ being selected becomes 0 MAZ_size_terms["size_term"] = np.where( From aa8d5a3b88b6ab3a4064dfe27786f1d919a6371f Mon Sep 17 00:00:00 2001 From: David Hensle Date: Fri, 4 Nov 2022 16:53:44 -0700 Subject: [PATCH 24/27] adding logic to skip external location choice models --- activitysim/abm/tables/shadow_pricing.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 8a3310819..569b9185d 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -140,6 +140,16 @@ def __init__( "shadow_settings %s: %s" % (k, self.shadow_settings.get(k)) ) + if ( + self.model_selector not in ["workplace", "school"] + and self.shadow_settings["SHADOW_PRICE_METHOD"] == "simulation" + ): + logger.warning( + "Shadow price simulation method is only implemented for workplace and school." + ) + logger.warning(f"Not using shadow pricing for {self.model_selector}") + self.use_shadow_pricing = False + # - destination_size_table (desired_size) self.desired_size = inject.get_table( size_table_name(self.model_selector) @@ -746,6 +756,10 @@ def update_shadow_prices(self): "", default_segment_to_name_dict ) segment_name = segment_to_name_dict[self.model_selector] + + if type(self.choices_synced) != pd.DataFrame: + self.choices_synced = self.choices_synced.to_frame() + choices_synced = self.choices_synced.merge( persons_merged[segment_name], how="left", From a79ac7b09fd6a553ddaedfdf47614548f0bd9c7a Mon Sep 17 00:00:00 2001 From: David Hensle Date: Wed, 14 Dec 2022 18:07:30 -0800 Subject: [PATCH 25/27] consistent multiprocessing results --- activitysim/abm/models/location_choice.py | 18 ----- activitysim/abm/tables/shadow_pricing.py | 50 +++++++----- .../configs/shadow_pricing.yaml | 6 +- .../test/configs/settings.yaml | 3 +- .../test/configs_mp/logging.yaml | 57 +++++++++++++ .../test/configs_mp/settings.yaml | 33 ++++++++ .../test/regress/final_trips.csv | 80 ++++++++++--------- .../test/test_mtc_extended.py | 78 ++++++++++++------ 8 files changed, 220 insertions(+), 105 deletions(-) create mode 100644 activitysim/examples/prototype_mtc_extended/test/configs_mp/logging.yaml create mode 100644 activitysim/examples/prototype_mtc_extended/test/configs_mp/settings.yaml diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index 5abf8f26c..c6e582d6c 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -907,24 +907,6 @@ def iterate_location_choice( persons_merged_df_ = persons_merged_df_[ persons_merged_df_.index.isin(spc.sampled_persons.index) ] - # handle cases where a segment has persons but no zones to receive them - desired_size_sum = spc.desired_size[ - spc.desired_size.index.isin( - spc.shadow_prices[spc.shadow_prices.iloc[:, 0] != -999].index - ) - ].sum() - zero_desired_size_segments = [ - i for i in desired_size_sum.index if desired_size_sum[i] == 0 - ] - zero_desired_size_segments_ids = [ - segment_ids[key] for key in zero_desired_size_segments - ] - persons_merged_df_ = persons_merged_df_[ - ~persons_merged_df_[chooser_segment_column].isin( - zero_desired_size_segments_ids - ) - ] - persons_merged_df_ = persons_merged_df_.sort_index() choices_df_, save_sample_df = run_location_choice( diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 569b9185d..9999a4a46 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -12,6 +12,7 @@ from activitysim.abm.tables.size_terms import tour_destination_size_terms from activitysim.core import config, inject, tracing, util from activitysim.core.input import read_input_table +from activitysim.core import logit logger = logging.getLogger(__name__) @@ -141,7 +142,8 @@ def __init__( ) if ( - self.model_selector not in ["workplace", "school"] + self.use_shadow_pricing + and self.model_selector not in ["workplace", "school"] and self.shadow_settings["SHADOW_PRICE_METHOD"] == "simulation" ): logger.warning( @@ -685,6 +687,7 @@ def update_shadow_prices(self): new_shadow_prices.where( self.modeled_size > 0, self.shadow_prices, inplace=True ) + self.shadow_prices = new_shadow_prices elif shadow_price_method == "daysim": # - Daysim @@ -732,6 +735,7 @@ def update_shadow_prices(self): ) new_shadow_prices = self.shadow_prices + adjustment + self.shadow_prices = new_shadow_prices elif shadow_price_method == "simulation": # - NewMethod @@ -808,22 +812,22 @@ def update_shadow_prices(self): & (choices_synced["segment"] == self.segment_ids[segment]) ]["choice"] - choices_index = choices.index.name - choices = choices.reset_index() - - # handling unlikely cases where there are no more overassigned zones, but a few underassigned zones remain - if len(choices) > 0: - current_sample = ( - choices.groupby("choice") - .apply( - # FIXME is this sample stable? - lambda x: x.sample( - frac=zonal_sample_rate.loc[x.name], random_state=1 - ) - ) - .reset_index(drop=True) - .set_index(choices_index) + # segment is converged if all zones are overpredicted / within tolerance + # do not want people to be re-simulated if no open zone exists + converged = len(overpredicted_zones) == len(self.shadow_prices) + + # draw persons assigned to overassigned zones to re-simulate if not converged + if (len(choices) > 0) & (~converged): + # person's probability of being selected for re-simulation is from the zonal sample rate + sample_rates = choices.map(zonal_sample_rate.to_dict()) + probs = pd.DataFrame( + data={'0': 1 - sample_rates, '1': sample_rates}, + index=choices.index ) + # using ActivitySim's RNG to make choices for repeatability + current_sample, rands = logit.make_choices(probs) + current_sample = current_sample[current_sample == 1] + if len(sampled_persons) == 0: sampled_persons = current_sample else: @@ -834,8 +838,6 @@ def update_shadow_prices(self): else: raise RuntimeError("unknown SHADOW_PRICE_METHOD %s" % shadow_price_method) - # self.shadow_prices = new_shadow_prices - def dest_size_terms(self, segment): assert segment in self.segment_ids @@ -1305,9 +1307,19 @@ def add_size_tables(): segment_scale_factors[c], ) ) + # FIXME - can get zero size if model_settings["CHOOSER_FILTER_COLUMN_NAME"] not yet determined / initialized to 0 + # using raw size if scaled size is 0. Is this an acceptable fix? + # this is happening for external models where extenal identification is not run yet at this stage + if segment_scale_factors[c] <= 0: + logger.warning( + f"scale_factor is <= 0 for {model_selector}:{c}, using raw size instead" + ) + segment_scale_factors[c] = 1 # FIXME - should we be rounding? - scaled_size = (raw_size * segment_scale_factors).round() + # scaled_size = (raw_size * segment_scale_factors).round() + # rounding can cause zero probability errors for small sample sizes + scaled_size = (raw_size * segment_scale_factors) else: scaled_size = raw_size diff --git a/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml b/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml index 593a36478..a8c1b62c6 100644 --- a/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml +++ b/activitysim/examples/prototype_mtc_extended/configs/shadow_pricing.yaml @@ -4,10 +4,11 @@ shadow_pricing_models: # global switch to enable/disable loading of saved shadow prices # (ignored if global use_shadow_pricing switch is False) +# (warm start not available for shadow price simulation method) LOAD_SAVED_SHADOW_PRICES: False # write out choices by iteration to trace folder -WRITE_ITERATION_CHOICES: True +WRITE_ITERATION_CHOICES: False # number of shadow price iterations for cold start MAX_ITERATIONS: 10 @@ -20,7 +21,6 @@ MAX_ITERATIONS_SAVED: 1 # SHADOW_PRICE_METHOD: daysim SHADOW_PRICE_METHOD: simulation -# --- simulation method settings # ignore criteria for zones smaller than size_threshold SIZE_THRESHOLD: 10 # ignore criteria for zones smaller than target_threshold (total employmnet or enrollment) @@ -29,6 +29,8 @@ TARGET_THRESHOLD: 20 PERCENT_TOLERANCE: 5 # max percentage of zones allowed to fail FAIL_THRESHOLD: 1 + +# --- simulation method settings # apply different targets for each segment specified in destination_size_terms.csv school_segmentation_targets: # format is segment: land_use_column diff --git a/activitysim/examples/prototype_mtc_extended/test/configs/settings.yaml b/activitysim/examples/prototype_mtc_extended/test/configs/settings.yaml index 78bdcf922..d62c82b8d 100644 --- a/activitysim/examples/prototype_mtc_extended/test/configs/settings.yaml +++ b/activitysim/examples/prototype_mtc_extended/test/configs/settings.yaml @@ -11,9 +11,8 @@ chunk_size: 0 # - shadow pricing global switches # turn shadow_pricing on and off for all models (e.g. school and work) -# shadow pricing is deprecated for less than full samples # see shadow_pricing.yaml for additional settings -use_shadow_pricing: False +use_shadow_pricing: True # turn writing of sample_tables on and off for all models # (if True, tables will be written if DEST_CHOICE_SAMPLE_TABLE_NAME is specified in individual model settings) diff --git a/activitysim/examples/prototype_mtc_extended/test/configs_mp/logging.yaml b/activitysim/examples/prototype_mtc_extended/test/configs_mp/logging.yaml new file mode 100644 index 000000000..e932009c5 --- /dev/null +++ b/activitysim/examples/prototype_mtc_extended/test/configs_mp/logging.yaml @@ -0,0 +1,57 @@ +# Config for logging +# ------------------ +# See http://docs.python.org/2.7/library/logging.config.html#configuration-dictionary-schema + +logging: + version: 1 + disable_existing_loggers: true + + + # Configuring the default (root) logger is highly recommended + root: + level: DEBUG + handlers: [console, logfile] + + loggers: + + activitysim: + level: DEBUG + handlers: [console, logfile] + propagate: false + + orca: + level: WARNING + handlers: [console, logfile] + propagate: false + + handlers: + + logfile: + class: logging.FileHandler + filename: !!python/object/apply:activitysim.core.config.log_file_path ['activitysim.log'] + mode: w + formatter: fileFormatter + level: NOTSET + + console: + class: logging.StreamHandler + stream: ext://sys.stdout + formatter: simpleFormatter + #level: NOTSET + level: !!python/object/apply:activitysim.core.mp_tasks.if_sub_task [WARNING, NOTSET] + + formatters: + + simpleFormatter: + class: logging.Formatter + #format: '%(processName)-10s %(levelname)s - %(name)s - %(message)s' + format: !!python/object/apply:activitysim.core.mp_tasks.if_sub_task [ + '%(processName)-10s %(levelname)s - %(name)s - %(message)s', + '%(levelname)s - %(name)s - %(message)s'] + datefmt: '%d/%m/%Y %H:%M:%S' + + fileFormatter: + class: logging.Formatter + format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s' + datefmt: '%d/%m/%Y %H:%M:%S' + diff --git a/activitysim/examples/prototype_mtc_extended/test/configs_mp/settings.yaml b/activitysim/examples/prototype_mtc_extended/test/configs_mp/settings.yaml new file mode 100644 index 000000000..229bfecef --- /dev/null +++ b/activitysim/examples/prototype_mtc_extended/test/configs_mp/settings.yaml @@ -0,0 +1,33 @@ +inherit_settings: True + +# number of households to simulate +households_sample_size: 10 + +strict: True + +chunk_size: 0 +multiprocess: True +num_processes: 2 + +# - shadow pricing global switches +# turn shadow_pricing on and off for all models (e.g. school and work) +# see shadow_pricing.yaml for additional settings +use_shadow_pricing: True + +# turn writing of sample_tables on and off for all models +# (if True, tables will be written if DEST_CHOICE_SAMPLE_TABLE_NAME is specified in individual model settings) +want_dest_choice_sample_tables: False + +# global switch to turn on or off presampling of destination alternatives at TAZ level (multizone models only) +want_dest_choice_presampling: True + +cleanup_pipeline_after_run: True + +output_tables: + h5_store: False + action: include + prefix: final_ + sort: True + tables: + - trips + - vehicles diff --git a/activitysim/examples/prototype_mtc_extended/test/regress/final_trips.csv b/activitysim/examples/prototype_mtc_extended/test/regress/final_trips.csv index 2260ceead..d238b3af7 100644 --- a/activitysim/examples/prototype_mtc_extended/test/regress/final_trips.csv +++ b/activitysim/examples/prototype_mtc_extended/test/regress/final_trips.csv @@ -5,26 +5,26 @@ trip_id,person_id,household_id,primary_purpose,trip_num,outbound,trip_count,dest 211388333,644476,386761,othdiscr,1,False,1,16,16,26423541,home,,18,WALK,7.330879513166791 211388353,644476,386761,othmaint,1,True,1,13,16,26423544,othmaint,,18,WALK,-0.4192505336997586 211388357,644476,386761,othmaint,1,False,1,16,13,26423544,home,,19,WALK,-0.41925030619506426 -211388441,644476,386761,work,1,True,1,4,16,26423555,work,,7,SHARED2FREE,0.6902839738239953 -211388445,644476,386761,work,1,False,1,16,4,26423555,home,,17,SHARED3FREE,-0.060176139897114116 -211388721,644477,386761,shopping,1,True,1,16,16,26423590,shopping,,14,WALK,5.857443647205806 -211388725,644477,386761,shopping,1,False,1,16,16,26423590,home,,14,WALK,5.857443647205806 -211389033,644478,386761,school,1,True,1,20,16,26423629,school,,11,WALK_LOC,1.5300445825829465 -211389037,644478,386761,school,1,False,1,16,20,26423629,home,,19,WALK_LRF,3.76602489416099 +211388441,644476,386761,work,1,True,1,5,16,26423555,work,,7,WALK,3.1601556277526734 +211388445,644476,386761,work,1,False,1,16,5,26423555,home,,17,WALK,3.0545561863039348 +211388721,644477,386761,shopping,1,True,1,16,16,26423590,shopping,,14,WALK,7.330879513046803 +211388725,644477,386761,shopping,1,False,1,16,16,26423590,home,,15,WALK,7.330879513166791 +211389033,644478,386761,school,1,True,1,22,16,26423629,school,,8,WALK_LOC,1.1439151605607298 +211389037,644478,386761,school,1,False,1,16,22,26423629,home,,16,WALK,1.0639554271252156 515832417,1572659,763879,shopping,1,True,3,25,6,64479052,othmaint,40.895784325733594,7,WALK,12.896301701456215 515832418,1572659,763879,shopping,2,True,3,25,25,64479052,escort,40.13139614585728,12,WALK,13.621701652814899 515832419,1572659,763879,shopping,3,True,3,24,25,64479052,shopping,,17,WALK,3.0930067693134005 515832421,1572659,763879,shopping,1,False,3,25,24,64479052,shopping,38.41713719577139,18,WALK,3.0706867412992 515832422,1572659,763879,shopping,2,False,3,7,25,64479052,escort,59.6309483835455,20,WALK,12.807021629683366 515832423,1572659,763879,shopping,3,False,3,6,7,64479052,home,,20,WALK,14.258626224164276 -535620049,1632987,824207,work,1,True,1,4,18,66952506,work,,15,WALK_LOC,0.2992185756062765 -535620053,1632987,824207,work,1,False,1,18,4,66952506,home,,21,WALK,0.5557332757868809 -615236801,1875721,982875,work,1,True,1,10,16,76904600,work,,8,WALK_LOC,7.627291076037502 -615236805,1875721,982875,work,1,False,1,16,10,76904600,home,,18,WALK_LOC,7.6193935739287255 +535620049,1632987,824207,work,1,True,1,2,18,66952506,work,,15,WALK_LOC,-0.04547980849224868 +535620053,1632987,824207,work,1,False,1,18,2,66952506,home,,21,WALK,0.28793961660642414 +615236801,1875721,982875,work,1,True,1,12,16,76904600,work,,8,WALK,4.408583578622911 +615236805,1875721,982875,work,1,False,1,16,12,76904600,home,,18,WALK,4.401112972600159 615236865,1875722,982875,eatout,1,True,2,7,16,76904608,escort,33.332775271121825,10,WALK,12.852466196970816 -615236866,1875722,982875,eatout,2,True,2,14,7,76904608,eatout,,13,WALK,0.0679239371174617 -615236869,1875722,982875,eatout,1,False,1,16,14,76904608,home,,17,WALK,0.9383092208675533 -708171009,2159057,1099626,work,1,True,1,2,20,88521376,work,,7,WALK,-0.35400338649017565 +615236866,1875722,982875,eatout,2,True,2,14,7,76904608,eatout,,13,WALK,0.06792393711746149 +615236869,1875722,982875,eatout,1,False,1,16,14,76904608,home,,16,WALK,0.938309220867553 +708171009,2159057,1099626,work,1,True,1,2,20,88521376,work,,7,WALK,-0.3540033864901755 708171013,2159057,1099626,work,1,False,2,8,2,88521376,shopping,28.059656557964445,18,WALK,0.34307389812569966 708171014,2159057,1099626,work,2,False,2,20,8,88521376,home,,18,WALK_LOC,9.930931452887558 708171273,2159058,1099626,univ,1,True,1,9,20,88521409,univ,,15,WALK_LOC,10.081589126967758 @@ -32,42 +32,44 @@ trip_id,person_id,household_id,primary_purpose,trip_num,outbound,trip_count,dest 708171601,2159059,1099626,school,1,True,1,20,20,88521450,school,,8,WALK,2.001157626801728 708171605,2159059,1099626,school,1,False,1,20,20,88521450,home,,13,WALK,2.001157626801728 841877257,2566698,1196298,work,1,True,1,1,25,105234657,work,,6,WALK,0.5218384234138416 -841877261,2566698,1196298,work,1,False,1,25,1,105234657,home,,17,WALK_LOC,0.4855336440096438 -841877849,2566700,1196298,school,1,True,1,25,25,105234731,school,,7,WALK,12.824615869979219 -841877853,2566700,1196298,school,1,False,1,25,25,105234731,home,,15,WALK,12.824615869979219 -841878177,2566701,1196298,school,1,True,1,3,25,105234772,school,,8,WALK,8.979312480941104 -841878181,2566701,1196298,school,1,False,1,25,3,105234772,home,,13,WALK,8.979312481086987 -841878505,2566702,1196298,school,1,True,1,6,25,105234813,school,,12,WALK_LOC,11.709395865665709 -841878509,2566702,1196298,school,1,False,2,25,6,105234813,shopping,51.23589608925055,20,WALK_LOC,11.238325436501778 -841878510,2566702,1196298,school,2,False,2,25,25,105234813,home,,20,WALK,11.641815891720018 +841877261,2566698,1196298,work,1,False,1,25,1,105234657,home,,17,WALK_LOC,0.4855336440096437 +841877849,2566700,1196298,school,1,True,1,24,25,105234731,school,,7,WALK_LOC,2.6558477997467738 +841877853,2566700,1196298,school,1,False,1,25,24,105234731,home,,15,WALK_LOC,2.6344417189136524 +841878177,2566701,1196298,school,1,True,1,1,25,105234772,school,,8,WALK_LOC,-0.7798918703154758 +841878181,2566701,1196298,school,1,False,1,25,1,105234772,home,,12,WALK_LOC,-0.8827606673241135 +841878505,2566702,1196298,school,1,True,1,2,25,105234813,school,,11,WALK_LOC,-0.1628834607507176 +841878509,2566702,1196298,school,1,False,4,25,2,105234813,shopping,29.517520243044103,15,WALK_LOC,-0.19533192942514782 +841878510,2566702,1196298,school,2,False,4,7,25,105234813,escort,54.82998544881473,21,WALK_LOC,12.277365654386355 +841878511,2566702,1196298,school,3,False,4,7,7,105234813,escort,56.30242155895923,21,WALK,12.412237383347824 +841878512,2566702,1196298,school,4,False,4,25,7,105234813,home,,21,WALK_LOC,12.572656398958213 1004301497,3061894,1363467,shopping,1,True,1,20,24,125537687,shopping,,12,TNC_SHARED,0.09686480459546322 1004301501,3061894,1363467,shopping,1,False,1,24,20,125537687,home,,13,DRIVEALONEFREE,0.015792413826355586 1004301761,3061895,1363467,othdiscr,1,True,1,9,24,125537720,othdiscr,,17,WALK_HVY,11.684658026322639 1004301765,3061895,1363467,othdiscr,1,False,1,24,9,125537720,home,,19,WALK_LRF,11.49938905905555 1004301785,3061895,1363467,othmaint,1,True,1,7,24,125537723,othmaint,,15,WALK,8.131583343724042 1004301789,3061895,1363467,othmaint,1,False,1,24,7,125537723,home,,16,WALK,8.096583148991245 -1004301873,3061895,1363467,work,1,True,1,25,24,125537734,work,,6,WALK,10.08552703351978 -1004301877,3061895,1363467,work,1,False,1,24,25,125537734,home,,13,WALK,10.103127058632895 -1368289969,4171615,1810015,univ,1,True,1,12,16,171036246,univ,,13,WALK,4.061179288088942 -1368289973,4171615,1810015,univ,1,False,1,16,12,171036246,home,,13,WALK,4.06117926693834 +1004301873,3061895,1363467,work,1,True,1,2,24,125537734,work,,6,WALK,-0.11170571223054646 +1004301877,3061895,1363467,work,1,False,1,24,2,125537734,home,,13,WALK,-0.08531919902466034 +1368289969,4171615,1810015,univ,1,True,1,5,16,171036246,univ,,13,WALK,3.565506569586174 +1368289973,4171615,1810015,univ,1,False,1,16,5,171036246,home,,13,WALK,3.489626616165674 1368290273,4171616,1810015,othmaint,1,True,1,3,16,171036284,othmaint,,9,WALK,5.752949863933666 1368290277,4171616,1810015,othmaint,1,False,1,16,3,171036284,home,,11,WALK,5.595449988691705 -1368290689,4171617,1810015,work,1,True,2,8,16,171036336,social,23.477345398553453,9,WALK,7.896576327414681 -1368290690,4171617,1810015,work,2,True,2,15,8,171036336,work,,12,WALK,-0.8211572450364255 -1368290693,4171617,1810015,work,1,False,1,16,15,171036336,home,,18,WALK,0.23912905823533456 +1368290689,4171617,1810015,work,1,True,2,8,16,171036336,social,23.627203658915438,9,WALK,7.896576327414681 +1368290690,4171617,1810015,work,2,True,2,2,8,171036336,work,,12,WALK,-0.6312570585756764 +1368290693,4171617,1810015,work,1,False,1,16,2,171036336,home,,18,WALK,-0.4904505209392121 1368291297,4171619,1810015,shopping,1,True,1,1,16,171036412,shopping,,13,WALK,-1.0053143437541998 1368291301,4171619,1810015,shopping,1,False,2,7,1,171036412,shopping,29.48393750646727,14,WALK,-1.5297238905680486 1368291302,4171619,1810015,shopping,2,False,2,16,7,171036412,home,,15,WALK,12.573466151788852 -1368291609,4171620,1810015,school,1,True,1,8,16,171036451,school,,7,WALK_LOC,10.68060996983474 -1368291613,4171620,1810015,school,1,False,1,16,8,171036451,home,,15,WALK_LOC,10.665116381563836 +1368291609,4171620,1810015,school,1,True,1,4,16,171036451,school,,7,WALK,-0.48035583077478106 +1368291613,4171620,1810015,school,1,False,1,16,4,171036451,home,,15,WALK,-0.48035850407554204 1368292281,4171622,1810015,shopping,1,True,1,19,16,171036535,shopping,,9,WALK,-2.394141994327624 1368292285,4171622,1810015,shopping,1,False,1,16,19,171036535,home,,15,WALK,-2.4219133842589526 -1368292377,4171623,1810015,atwork,1,True,1,7,21,171036547,atwork,,10,WALK,13.897946303660285 -1368292381,4171623,1810015,atwork,1,False,2,6,7,171036547,othmaint,62.239483838845736,10,WALK,14.364186248721689 -1368292382,4171623,1810015,atwork,2,False,2,21,6,171036547,work,,10,WALK,12.200629295549986 -1368292657,4171623,1810015,work,1,True,2,25,16,171036582,escort,30.234430836012045,8,WALK,9.029527074456235 -1368292658,4171623,1810015,work,2,True,2,21,25,171036582,work,,8,WALK,2.0014416307382055 -1368292661,4171623,1810015,work,1,False,2,7,21,171036582,work,34.72578612209499,23,WALK,2.5646380272501808 +1368292377,4171623,1810015,atwork,1,True,1,5,22,171036547,atwork,,10,WALK,4.427779849728023 +1368292381,4171623,1810015,atwork,1,False,2,6,5,171036547,othmaint,43.14463209840464,10,WALK,4.690979681288974 +1368292382,4171623,1810015,atwork,2,False,2,22,6,171036547,work,,10,WALK,11.531029342444546 +1368292657,4171623,1810015,work,1,True,2,25,16,171036582,escort,27.0134960004981,8,WALK,9.029527074456235 +1368292658,4171623,1810015,work,2,True,2,22,25,171036582,work,,8,WALK,0.6659789002288896 +1368292661,4171623,1810015,work,1,False,2,7,22,171036582,work,29.869621267785206,23,WALK,-0.09956819366821217 1368292662,4171623,1810015,work,2,False,2,16,7,171036582,home,,23,WALK,9.584561374619746 2464104641,7512514,2821179,eatout,1,True,1,13,8,308013080,eatout,,8,WALK,-1.171759971785514 2464104645,7512514,2821179,eatout,1,False,1,8,13,308013080,home,,16,WALK,-1.238718768693438 @@ -75,6 +77,6 @@ trip_id,person_id,household_id,primary_purpose,trip_num,outbound,trip_count,dest 2464104861,7512514,2821179,shopping,1,False,1,8,6,308013107,home,,19,WALK,12.322088998148224 2464104881,7512514,2821179,social,1,True,1,9,8,308013110,social,,16,WALK,6.292424410910544 2464104885,7512514,2821179,social,1,False,1,8,9,308013110,home,,16,WALK_LOC,6.322192231184283 -2464449633,7513565,2822230,work,1,True,2,9,8,308056204,univ,40.040196758213916,9,WALK,7.9948426686587775 -2464449634,7513565,2822230,work,2,True,2,25,9,308056204,work,,9,WALK,8.34752700809022 -2464449637,7513565,2822230,work,1,False,1,8,25,308056204,home,,18,WALK,9.31552710851258 +2464449633,7513565,2822230,work,1,True,2,9,8,308056204,univ,22.003640452037526,10,WALK,7.994842668604955 +2464449634,7513565,2822230,work,2,True,2,2,9,308056204,work,,10,WALK,-1.58478835829668 +2464449637,7513565,2822230,work,1,False,1,8,2,308056204,home,,19,WALK,-0.7715462832235578 diff --git a/activitysim/examples/prototype_mtc_extended/test/test_mtc_extended.py b/activitysim/examples/prototype_mtc_extended/test/test_mtc_extended.py index cc7bb7b54..5e4176e64 100644 --- a/activitysim/examples/prototype_mtc_extended/test/test_mtc_extended.py +++ b/activitysim/examples/prototype_mtc_extended/test/test_mtc_extended.py @@ -15,7 +15,7 @@ def teardown_function(func): inject.reinject_decorated_tables() -def test_prototype_mtc_extended(): +def run_test_prototype_mtc_extended(multiprocess=False): def example_path(dirname): resource = os.path.join("examples", "prototype_mtc_extended", dirname) return pkg_resources.resource_filename("activitysim", resource) @@ -34,37 +34,65 @@ def regress(): regress_vehicles_df = pd.read_csv(test_path("regress/final_vehicles.csv")) final_vehicles_df = pd.read_csv(test_path("output/final_vehicles.csv")) - # person_id,household_id,tour_id,primary_purpose,trip_num,outbound,trip_count,purpose, - # destination,origin,destination_logsum,depart,trip_mode,mode_choice_logsum - # compare_cols = [] pdt.assert_frame_equal(final_trips_df, regress_trips_df) pdt.assert_frame_equal(final_vehicles_df, regress_vehicles_df) file_path = os.path.join(os.path.dirname(__file__), "simulation.py") - subprocess.run( - [ - "coverage", - "run", - "-a", - file_path, - "-c", - test_path("configs"), - "-c", - example_path("configs"), - "-c", - example_mtc_path("configs"), - "-d", - example_mtc_path("data"), - "-o", - test_path("output"), - ], - check=True, - ) + if multiprocess: + subprocess.run( + [ + "coverage", + "run", + "-a", + file_path, + "-c", + test_path("configs_mp"), + "-c", + example_path("configs_mp"), + "-c", + example_path("configs"), + "-c", + example_mtc_path("configs"), + "-d", + example_mtc_path("data"), + "-o", + test_path("output"), + ], + check=True, + ) + else: + subprocess.run( + [ + "coverage", + "run", + "-a", + file_path, + "-c", + test_path("configs"), + "-c", + example_path("configs"), + "-c", + example_mtc_path("configs"), + "-d", + example_mtc_path("data"), + "-o", + test_path("output"), + ], + check=True, + ) regress() -if __name__ == "__main__": +def test_prototype_mtc_extended(): + run_test_prototype_mtc_extended(multiprocess=False) + - test_prototype_mtc_extended() +def test_prototype_mtc_extended_mp(): + run_test_prototype_mtc_extended(multiprocess=True) + + +if __name__ == "__main__": + run_test_prototype_mtc_extended(multiprocess=False) + run_test_prototype_mtc_extended(multiprocess=True) From 0d90f875a63a737f2270935a760f5d4fe95ed352 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Wed, 14 Dec 2022 18:17:30 -0800 Subject: [PATCH 26/27] blacken --- activitysim/abm/tables/shadow_pricing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/activitysim/abm/tables/shadow_pricing.py b/activitysim/abm/tables/shadow_pricing.py index 9999a4a46..a9e4a88a2 100644 --- a/activitysim/abm/tables/shadow_pricing.py +++ b/activitysim/abm/tables/shadow_pricing.py @@ -821,8 +821,8 @@ def update_shadow_prices(self): # person's probability of being selected for re-simulation is from the zonal sample rate sample_rates = choices.map(zonal_sample_rate.to_dict()) probs = pd.DataFrame( - data={'0': 1 - sample_rates, '1': sample_rates}, - index=choices.index + data={"0": 1 - sample_rates, "1": sample_rates}, + index=choices.index, ) # using ActivitySim's RNG to make choices for repeatability current_sample, rands = logit.make_choices(probs) @@ -1319,7 +1319,7 @@ def add_size_tables(): # FIXME - should we be rounding? # scaled_size = (raw_size * segment_scale_factors).round() # rounding can cause zero probability errors for small sample sizes - scaled_size = (raw_size * segment_scale_factors) + scaled_size = raw_size * segment_scale_factors else: scaled_size = raw_size From 3a78433232c8571a80e9f0583df65b65fccf14d4 Mon Sep 17 00:00:00 2001 From: David Hensle Date: Wed, 14 Dec 2022 23:44:55 -0800 Subject: [PATCH 27/27] updating regression trips --- .../test/regress/final_trips.csv | 148 ++++++++---------- 1 file changed, 67 insertions(+), 81 deletions(-) diff --git a/activitysim/examples/prototype_mtc_extended/test/regress/final_trips.csv b/activitysim/examples/prototype_mtc_extended/test/regress/final_trips.csv index d238b3af7..3ad350944 100644 --- a/activitysim/examples/prototype_mtc_extended/test/regress/final_trips.csv +++ b/activitysim/examples/prototype_mtc_extended/test/regress/final_trips.csv @@ -1,82 +1,68 @@ trip_id,person_id,household_id,primary_purpose,trip_num,outbound,trip_count,destination,origin,tour_id,purpose,destination_logsum,depart,trip_mode,mode_choice_logsum -211388201,644476,386761,escort,1,True,1,11,16,26423525,escort,,5,WALK_LOC,4.789158412380779 -211388205,644476,386761,escort,1,False,1,16,11,26423525,home,,6,WALK_LOC,5.050171287433508 -211388329,644476,386761,othdiscr,1,True,1,16,16,26423541,othdiscr,,18,WALK,7.330879513166791 -211388333,644476,386761,othdiscr,1,False,1,16,16,26423541,home,,18,WALK,7.330879513166791 -211388353,644476,386761,othmaint,1,True,1,13,16,26423544,othmaint,,18,WALK,-0.4192505336997586 -211388357,644476,386761,othmaint,1,False,1,16,13,26423544,home,,19,WALK,-0.41925030619506426 -211388441,644476,386761,work,1,True,1,5,16,26423555,work,,7,WALK,3.1601556277526734 -211388445,644476,386761,work,1,False,1,16,5,26423555,home,,17,WALK,3.0545561863039348 -211388721,644477,386761,shopping,1,True,1,16,16,26423590,shopping,,14,WALK,7.330879513046803 -211388725,644477,386761,shopping,1,False,1,16,16,26423590,home,,15,WALK,7.330879513166791 -211389033,644478,386761,school,1,True,1,22,16,26423629,school,,8,WALK_LOC,1.1439151605607298 -211389037,644478,386761,school,1,False,1,16,22,26423629,home,,16,WALK,1.0639554271252156 -515832417,1572659,763879,shopping,1,True,3,25,6,64479052,othmaint,40.895784325733594,7,WALK,12.896301701456215 -515832418,1572659,763879,shopping,2,True,3,25,25,64479052,escort,40.13139614585728,12,WALK,13.621701652814899 -515832419,1572659,763879,shopping,3,True,3,24,25,64479052,shopping,,17,WALK,3.0930067693134005 -515832421,1572659,763879,shopping,1,False,3,25,24,64479052,shopping,38.41713719577139,18,WALK,3.0706867412992 -515832422,1572659,763879,shopping,2,False,3,7,25,64479052,escort,59.6309483835455,20,WALK,12.807021629683366 -515832423,1572659,763879,shopping,3,False,3,6,7,64479052,home,,20,WALK,14.258626224164276 -535620049,1632987,824207,work,1,True,1,2,18,66952506,work,,15,WALK_LOC,-0.04547980849224868 -535620053,1632987,824207,work,1,False,1,18,2,66952506,home,,21,WALK,0.28793961660642414 -615236801,1875721,982875,work,1,True,1,12,16,76904600,work,,8,WALK,4.408583578622911 -615236805,1875721,982875,work,1,False,1,16,12,76904600,home,,18,WALK,4.401112972600159 -615236865,1875722,982875,eatout,1,True,2,7,16,76904608,escort,33.332775271121825,10,WALK,12.852466196970816 -615236866,1875722,982875,eatout,2,True,2,14,7,76904608,eatout,,13,WALK,0.06792393711746149 -615236869,1875722,982875,eatout,1,False,1,16,14,76904608,home,,16,WALK,0.938309220867553 -708171009,2159057,1099626,work,1,True,1,2,20,88521376,work,,7,WALK,-0.3540033864901755 -708171013,2159057,1099626,work,1,False,2,8,2,88521376,shopping,28.059656557964445,18,WALK,0.34307389812569966 -708171014,2159057,1099626,work,2,False,2,20,8,88521376,home,,18,WALK_LOC,9.930931452887558 -708171273,2159058,1099626,univ,1,True,1,9,20,88521409,univ,,15,WALK_LOC,10.081589126967758 -708171277,2159058,1099626,univ,1,False,1,20,9,88521409,home,,18,WALK_LOC,9.700222902924416 -708171601,2159059,1099626,school,1,True,1,20,20,88521450,school,,8,WALK,2.001157626801728 -708171605,2159059,1099626,school,1,False,1,20,20,88521450,home,,13,WALK,2.001157626801728 -841877257,2566698,1196298,work,1,True,1,1,25,105234657,work,,6,WALK,0.5218384234138416 -841877261,2566698,1196298,work,1,False,1,25,1,105234657,home,,17,WALK_LOC,0.4855336440096437 -841877849,2566700,1196298,school,1,True,1,24,25,105234731,school,,7,WALK_LOC,2.6558477997467738 -841877853,2566700,1196298,school,1,False,1,25,24,105234731,home,,15,WALK_LOC,2.6344417189136524 -841878177,2566701,1196298,school,1,True,1,1,25,105234772,school,,8,WALK_LOC,-0.7798918703154758 -841878181,2566701,1196298,school,1,False,1,25,1,105234772,home,,12,WALK_LOC,-0.8827606673241135 -841878505,2566702,1196298,school,1,True,1,2,25,105234813,school,,11,WALK_LOC,-0.1628834607507176 -841878509,2566702,1196298,school,1,False,4,25,2,105234813,shopping,29.517520243044103,15,WALK_LOC,-0.19533192942514782 -841878510,2566702,1196298,school,2,False,4,7,25,105234813,escort,54.82998544881473,21,WALK_LOC,12.277365654386355 -841878511,2566702,1196298,school,3,False,4,7,7,105234813,escort,56.30242155895923,21,WALK,12.412237383347824 -841878512,2566702,1196298,school,4,False,4,25,7,105234813,home,,21,WALK_LOC,12.572656398958213 -1004301497,3061894,1363467,shopping,1,True,1,20,24,125537687,shopping,,12,TNC_SHARED,0.09686480459546322 -1004301501,3061894,1363467,shopping,1,False,1,24,20,125537687,home,,13,DRIVEALONEFREE,0.015792413826355586 -1004301761,3061895,1363467,othdiscr,1,True,1,9,24,125537720,othdiscr,,17,WALK_HVY,11.684658026322639 -1004301765,3061895,1363467,othdiscr,1,False,1,24,9,125537720,home,,19,WALK_LRF,11.49938905905555 -1004301785,3061895,1363467,othmaint,1,True,1,7,24,125537723,othmaint,,15,WALK,8.131583343724042 -1004301789,3061895,1363467,othmaint,1,False,1,24,7,125537723,home,,16,WALK,8.096583148991245 -1004301873,3061895,1363467,work,1,True,1,2,24,125537734,work,,6,WALK,-0.11170571223054646 -1004301877,3061895,1363467,work,1,False,1,24,2,125537734,home,,13,WALK,-0.08531919902466034 -1368289969,4171615,1810015,univ,1,True,1,5,16,171036246,univ,,13,WALK,3.565506569586174 -1368289973,4171615,1810015,univ,1,False,1,16,5,171036246,home,,13,WALK,3.489626616165674 -1368290273,4171616,1810015,othmaint,1,True,1,3,16,171036284,othmaint,,9,WALK,5.752949863933666 -1368290277,4171616,1810015,othmaint,1,False,1,16,3,171036284,home,,11,WALK,5.595449988691705 -1368290689,4171617,1810015,work,1,True,2,8,16,171036336,social,23.627203658915438,9,WALK,7.896576327414681 -1368290690,4171617,1810015,work,2,True,2,2,8,171036336,work,,12,WALK,-0.6312570585756764 -1368290693,4171617,1810015,work,1,False,1,16,2,171036336,home,,18,WALK,-0.4904505209392121 -1368291297,4171619,1810015,shopping,1,True,1,1,16,171036412,shopping,,13,WALK,-1.0053143437541998 -1368291301,4171619,1810015,shopping,1,False,2,7,1,171036412,shopping,29.48393750646727,14,WALK,-1.5297238905680486 -1368291302,4171619,1810015,shopping,2,False,2,16,7,171036412,home,,15,WALK,12.573466151788852 -1368291609,4171620,1810015,school,1,True,1,4,16,171036451,school,,7,WALK,-0.48035583077478106 -1368291613,4171620,1810015,school,1,False,1,16,4,171036451,home,,15,WALK,-0.48035850407554204 -1368292281,4171622,1810015,shopping,1,True,1,19,16,171036535,shopping,,9,WALK,-2.394141994327624 -1368292285,4171622,1810015,shopping,1,False,1,16,19,171036535,home,,15,WALK,-2.4219133842589526 -1368292377,4171623,1810015,atwork,1,True,1,5,22,171036547,atwork,,10,WALK,4.427779849728023 -1368292381,4171623,1810015,atwork,1,False,2,6,5,171036547,othmaint,43.14463209840464,10,WALK,4.690979681288974 -1368292382,4171623,1810015,atwork,2,False,2,22,6,171036547,work,,10,WALK,11.531029342444546 -1368292657,4171623,1810015,work,1,True,2,25,16,171036582,escort,27.0134960004981,8,WALK,9.029527074456235 -1368292658,4171623,1810015,work,2,True,2,22,25,171036582,work,,8,WALK,0.6659789002288896 -1368292661,4171623,1810015,work,1,False,2,7,22,171036582,work,29.869621267785206,23,WALK,-0.09956819366821217 -1368292662,4171623,1810015,work,2,False,2,16,7,171036582,home,,23,WALK,9.584561374619746 -2464104641,7512514,2821179,eatout,1,True,1,13,8,308013080,eatout,,8,WALK,-1.171759971785514 -2464104645,7512514,2821179,eatout,1,False,1,8,13,308013080,home,,16,WALK,-1.238718768693438 -2464104857,7512514,2821179,shopping,1,True,1,6,8,308013107,shopping,,17,WALK,12.612248978887928 -2464104861,7512514,2821179,shopping,1,False,1,8,6,308013107,home,,19,WALK,12.322088998148224 -2464104881,7512514,2821179,social,1,True,1,9,8,308013110,social,,16,WALK,6.292424410910544 -2464104885,7512514,2821179,social,1,False,1,8,9,308013110,home,,16,WALK_LOC,6.322192231184283 -2464449633,7513565,2822230,work,1,True,2,9,8,308056204,univ,22.003640452037526,10,WALK,7.994842668604955 -2464449634,7513565,2822230,work,2,True,2,2,9,308056204,work,,10,WALK,-1.58478835829668 -2464449637,7513565,2822230,work,1,False,1,8,2,308056204,home,,19,WALK,-0.7715462832235578 +270680011,644476,386761,escort,1,True,1,16,16,27068001,escort,,5,WALK,0.5320114470722612 +270680016,644476,386761,escort,1,False,1,16,16,27068001,home,,5,TNC_SHARED,0.5320114470722612 +270680321,644476,386761,work,1,True,1,5,16,27068032,work,,5,WALK,3.784483617465789 +270680326,644476,386761,work,1,False,2,25,5,27068032,escort,36.837865536411556,17,WALK_LOC,3.8263633311364242 +270680327,644476,386761,work,2,False,2,16,25,27068032,home,,19,WALK,10.360438250702908 +270680681,644477,386761,shopping,1,True,1,5,16,27068068,shopping,,10,WALK,4.08926155146542 +270680686,644477,386761,shopping,1,False,1,16,5,27068068,home,,19,WALK,4.011141788832443 +270681081,644478,386761,school,1,True,1,22,16,27068108,school,,7,WALK,0.4463126906111215 +270681086,644478,386761,school,1,False,1,16,22,27068108,home,,13,WALK,0.6143299273673037 +660517121,1572659,763879,shopping,1,True,2,9,6,66051712,shopping,30.034881810916566,11,WALK_LOC,10.793454218489755 +660517122,1572659,763879,shopping,2,True,2,19,9,66051712,shopping,,12,TNC_SINGLE,0.5947152742072429 +660517126,1572659,763879,shopping,1,False,1,6,19,66051712,home,,20,WALK_LOC,0.40082631615857406 +685854941,1632987,824207,work,1,True,1,2,18,68585494,work,,7,WALK_LOC,0.012358327110380882 +685854946,1632987,824207,work,1,False,1,18,2,68585494,home,,17,WALK_LRF,0.3476878505929388 +787803221,1875721,982875,work,1,True,1,12,16,78780322,work,,8,BIKE,3.3186538058635886 +787803226,1875721,982875,work,1,False,1,16,12,78780322,home,,17,BIKE,3.3186537522718957 +787803301,1875722,982875,eatout,1,True,1,18,16,78780330,eatout,,7,WALK,0.25375271650786135 +787803306,1875722,982875,eatout,1,False,1,16,18,78780330,home,,13,WALK,0.22587009469786754 +906804341,2159057,1099626,work,1,True,1,2,20,90680434,work,,6,WALK,-0.31801945845680446 +906804346,2159057,1099626,work,1,False,1,20,2,90680434,home,,16,DRIVEALONEFREE,-2.1064263496483453 +906804681,2159058,1099626,univ,1,True,1,9,20,90680468,univ,,17,WALK_LOC,10.081589126967758 +906804686,2159058,1099626,univ,1,False,1,20,9,90680468,home,,20,WALK_LOC,9.95195853064744 +906805101,2159059,1099626,school,1,True,1,20,20,90680510,school,,7,WALK,2.001157626801728 +906805106,2159059,1099626,school,1,False,1,20,20,90680510,home,,15,WALK,2.001157626801728 +1078013561,2566698,1196298,work,1,True,1,1,25,107801356,work,,6,TNC_SINGLE,0.5218384234138416 +1078013566,2566698,1196298,work,1,False,1,25,1,107801356,home,,16,TNC_SINGLE,0.48556944294283877 +1078014321,2566700,1196298,school,1,True,1,24,25,107801432,school,,7,WALK_LOC,2.6558477997467738 +1078014326,2566700,1196298,school,1,False,1,25,24,107801432,home,,9,WALK_LOC,2.5642389027598185 +1078014741,2566701,1196298,school,1,True,1,1,25,107801474,school,,8,WALK_LOC,-0.7798918703154758 +1078014746,2566701,1196298,school,1,False,1,25,1,107801474,home,,16,WALK_LOC,-0.8462395764506365 +1078015161,2566702,1196298,school,1,True,1,2,25,107801516,school,,8,WALK,-0.12577374954461829 +1078015166,2566702,1196298,school,1,False,1,25,2,107801516,home,,13,WALK,-0.3303965444468694 +1285995821,3061894,1363467,shopping,1,True,2,7,24,128599582,othmaint,36.460758096403005,8,WALK,12.852372712730944 +1285995822,3061894,1363467,shopping,2,True,2,19,7,128599582,shopping,,8,DRIVEALONEFREE,0.6685799294322372 +1285995826,3061894,1363467,shopping,1,False,1,24,19,128599582,home,,19,SHARED2FREE,-0.6841449484072673 +1285996191,3061895,1363467,othmaint,1,True,1,4,24,128599619,othmaint,,7,WALK_LOC,0.8657906793525643 +1285996196,3061895,1363467,othmaint,1,False,1,24,4,128599619,home,,15,TNC_SINGLE,0.8645860484799108 +1285996271,3061895,1363467,social,1,True,1,11,24,128599627,social,,16,WALK_LOC,3.2107016897939977 +1285996276,3061895,1363467,social,1,False,1,24,11,128599627,home,,16,WALK_LOC,3.1342659069708274 +1285996301,3061895,1363467,work,1,True,1,2,24,128599630,work,,16,WALK,-0.11170571223054646 +1285996306,3061895,1363467,work,1,False,1,24,2,128599630,home,,22,WALK,-0.0853172208777054 +1752078621,4171615,1810015,univ,1,True,3,8,16,175207862,shopping,35.03505555221861,7,WALK,9.727146483995874 +1752078622,4171615,1810015,univ,2,True,3,7,8,175207862,shopping,41.47430148005939,8,WALK,13.486637386410468 +1752078623,4171615,1810015,univ,3,True,3,5,7,175207862,univ,,8,WALK,4.07498648453161 +1752078626,4171615,1810015,univ,1,False,1,16,5,175207862,home,,13,WALK,3.489626616165674 +1752079011,4171616,1810015,othmaint,1,True,1,19,16,175207901,othmaint,,14,TNC_SHARED,0.0016175329783967448 +1752079016,4171616,1810015,othmaint,1,False,1,16,19,175207901,home,,20,DRIVEALONEFREE,-0.9029931043506836 +1752079181,4171617,1810015,atwork,1,True,1,24,2,175207918,atwork,,14,WALK,3.35042938901452 +1752079186,4171617,1810015,atwork,1,False,1,2,24,175207918,work,,14,WALK,3.327869428854691 +1752079541,4171617,1810015,work,1,True,1,2,16,175207954,work,,9,WALK,-0.3408543940448438 +1752079546,4171617,1810015,work,1,False,1,16,2,175207954,home,,18,WALK,-0.4904505209392121 +1752080321,4171619,1810015,shopping,1,True,1,1,16,175208032,shopping,,14,WALK,-1.0053143437541998 +1752080326,4171619,1810015,shopping,1,False,1,16,1,175208032,home,,15,WALK,-1.1950043076874513 +1752080721,4171620,1810015,school,1,True,1,4,16,175208072,school,,7,WALK,-0.48035583077478106 +1752080726,4171620,1810015,school,1,False,1,16,4,175208072,home,,16,WALK,-0.48035850407554204 +1752081581,4171622,1810015,shopping,1,True,1,16,16,175208158,shopping,,11,WALK,7.3308794494701575 +1752081586,4171622,1810015,shopping,1,False,1,16,16,175208158,home,,15,WALK,7.330879449572421 +1752081701,4171623,1810015,atwork,1,True,1,1,22,175208170,atwork,,12,WALK,-0.6542981934820419 +1752081706,4171623,1810015,atwork,1,False,1,22,1,175208170,work,,13,WALK,-0.9475935966647256 +1752082061,4171623,1810015,work,1,True,1,22,16,175208206,work,,7,WALK,0.362388299138621 +1752082066,4171623,1810015,work,1,False,1,16,22,175208206,home,,22,WALK,0.4987831386861982 +3155256171,7512514,2821179,othmaint,1,True,1,9,8,315525617,othmaint,,12,WALK,6.622033921399732 +3155256176,7512514,2821179,othmaint,1,False,1,8,9,315525617,home,,15,WALK,6.622033921399732 +3155256221,7512514,2821179,shopping,1,True,1,2,8,315525622,shopping,,12,WALK,0.12220973142838766 +3155256226,7512514,2821179,shopping,1,False,1,8,2,315525622,home,,12,BIKE,0.06782315574947502 +3155697701,7513565,2822230,work,1,True,1,2,8,315569770,work,,8,WALK_LOC,0.38883325086589965 +3155697706,7513565,2822230,work,1,False,1,8,2,315569770,home,,21,WALK_LRF,0.741826248790529