From f4f79393cf5e00e5f5c97609a579f6964d983038 Mon Sep 17 00:00:00 2001 From: rakow Date: Mon, 4 Nov 2024 16:02:37 +0100 Subject: [PATCH 1/3] write mode chains in trip analysis --- .../analysis/population/TripAnalysis.java | 89 ++++++++++++++++--- 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java index 893c05cf9fd..8320d4aa97d 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java @@ -4,10 +4,7 @@ import it.unimi.dsi.fastutil.ints.IntList; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.ints.IntSet; -import it.unimi.dsi.fastutil.objects.Object2IntLinkedOpenHashMap; -import it.unimi.dsi.fastutil.objects.Object2IntMap; -import it.unimi.dsi.fastutil.objects.Object2LongMap; -import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; +import it.unimi.dsi.fastutil.objects.*; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.math3.analysis.interpolation.LoessInterpolator; @@ -37,6 +34,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static tech.tablesaw.aggregate.AggregateFunctions.count; @@ -47,7 +45,7 @@ produces = { "mode_share.csv", "mode_share_per_dist.csv", "mode_users.csv", "trip_stats.csv", "mode_share_per_%s.csv", "population_trip_stats.csv", "trip_purposes_by_hour.csv", - "mode_share_distance_distribution.csv", "mode_shift.csv", + "mode_share_distance_distribution.csv", "mode_shift.csv", "mode_chains.csv", "mode_choices.csv", "mode_choice_evaluation.csv", "mode_choice_evaluation_per_mode.csv", "mode_confusion_matrix.csv", "mode_prediction_error.csv" } @@ -305,17 +303,23 @@ public Integer call() throws Exception { writePopulationStats(persons, joined); - writeTripStats(joined); - - writeTripPurposes(joined); - - writeTripDistribution(joined); - - writeModeShift(joined); + tryRun(this::writeTripStats, joined); + tryRun(this::writeTripPurposes, joined); + tryRun(this::writeTripDistribution, joined); + tryRun(this::writeModeShift, joined); + tryRun(this::writeModeChains, joined); return 0; } + private void tryRun(ThrowingConsumer f, Table df) { + try { + f.accept(df); + } catch (IOException e) { + log.error("Error while running method", e); + } + } + private void writeModeShare(Table trips, List labels) { Table aggr = trips.summarize("trip_id", count).by("dist_group", "main_mode"); @@ -610,6 +614,62 @@ private void writeModeShift(Table trips) throws IOException { aggr.write().csv(output.getPath("mode_shift.csv").toFile()); } + /** + * Collects information about all modes used during one day. + */ + private void writeModeChains(Table trips) throws IOException { + + Map> modesPerPerson = new LinkedHashMap<>(); + + for (Row trip : trips) { + String id = trip.getString("person"); + String mode = trip.getString("main_mode"); + modesPerPerson.computeIfAbsent(id, s -> new LinkedList<>()).add(mode); + } + + // Store other values explicitly + ObjectDoubleMutablePair other = ObjectDoubleMutablePair.of("other", 0); + Object2DoubleMap chains = new Object2DoubleOpenHashMap<>(); + for (List modes : modesPerPerson.values()) { + String key; + if (modes.size() == 1) + key = modes.getFirst(); + else if (modes.size() > 6) { + other.right(other.rightDouble() + 1); + continue; + } else + key = String.join("-", modes); + + chains.mergeDouble(key, 1, Double::sum); + } + + + List> counts = chains.object2DoubleEntrySet().stream() + .map(e -> ObjectDoubleMutablePair.of(e.getKey(), (int) e.getDoubleValue())) + .sorted(Comparator.comparingDouble(p -> -p.rightDouble())) + .collect(Collectors.toList()); + + // Aggregate entries to prevent file from getting too large + for (int i = 250; i < counts.size(); i++) { + other.right(other.rightDouble() + counts.get(i).rightDouble()); + } + counts = counts.subList(0, Math.min(counts.size(), 250)); + counts.add(other); + + counts.sort(Comparator.comparingDouble(p -> -p.rightDouble())); + + + try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mode_chains.csv")), CSVFormat.DEFAULT)) { + + printer.printRecord("modes", "count", "share"); + + double total = counts.stream().mapToDouble(ObjectDoubleMutablePair::rightDouble).sum(); + for (ObjectDoubleMutablePair p : counts) { + printer.printRecord(p.left(), (int) p.rightDouble(), p.rightDouble() / total); + } + } + } + /** * How shape file filtering should be applied. */ @@ -619,4 +679,9 @@ enum LocationFilter { home, none } + + @FunctionalInterface + private interface ThrowingConsumer { + void accept(T t) throws IOException; + } } From 3dd881edc85e3a600937237e1fca9d47301b68ef Mon Sep 17 00:00:00 2001 From: rakow Date: Tue, 5 Nov 2024 09:45:51 +0100 Subject: [PATCH 2/3] add output to write mode shares per purpose --- .../application/analysis/population/TripAnalysis.java | 6 ++++-- .../analysis/population/TripByGroupAnalysis.java | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java index 8320d4aa97d..74c26ded21a 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java @@ -44,7 +44,8 @@ requires = {"trips.csv", "persons.csv"}, produces = { "mode_share.csv", "mode_share_per_dist.csv", "mode_users.csv", "trip_stats.csv", - "mode_share_per_%s.csv", "population_trip_stats.csv", "trip_purposes_by_hour.csv", + "mode_share_per_%s.csv", "mode_share_per_purpose_per_%s.csv", + "population_trip_stats.csv", "trip_purposes_by_hour.csv", "mode_share_distance_distribution.csv", "mode_shift.csv", "mode_chains.csv", "mode_choices.csv", "mode_choice_evaluation.csv", "mode_choice_evaluation_per_mode.csv", "mode_confusion_matrix.csv", "mode_prediction_error.csv" @@ -284,7 +285,8 @@ public Integer call() throws Exception { writeModeShare(joined, labels); if (groups != null) { - groups.analyzeModeShare(joined, labels, modeOrder, (g) -> output.getPath("mode_share_per_%s.csv", g)); + groups.writeModeShare(joined, labels, modeOrder, (g) -> output.getPath("mode_share_per_%s.csv", g)); + groups.writeModeSharePerPurpose(joined,modeOrder, (g) -> output.getPath("mode_share_per_purpose_per_%s.csv", g)); } if (persons.containsColumn(ATTR_REF_MODES)) { diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java index b63a58f5ed8..9d0ffaaac26 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java @@ -100,7 +100,7 @@ final class TripByGroupAnalysis { } } - void analyzeModeShare(Table trips, List dists, List modeOrder, Function output) { + void writeModeShare(Table trips, List dists, List modeOrder, Function output) { for (Group group : groups) { @@ -147,6 +147,12 @@ void analyzeModeShare(Table trips, List dists, List modeOrder, F } } + void writeModeSharePerPurpose(Table trips, List modeOrder, Function output) { + + // TODO: implement + + } + void groupPersons(Table persons) { for (Group g : groups) { From d388036c01a9c80cafe1c029a46bcd4ab07776fd Mon Sep 17 00:00:00 2001 From: rakow Date: Tue, 5 Nov 2024 14:45:01 +0100 Subject: [PATCH 3/3] write mode share per purpose --- .../analysis/population/TripAnalysis.java | 42 +++++++++++++++---- .../population/TripByGroupAnalysis.java | 6 --- .../simwrapper/dashboard/DashboardTests.java | 1 + 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java index 74c26ded21a..2365ed8a189 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripAnalysis.java @@ -24,6 +24,7 @@ import org.matsim.core.utils.io.IOUtils; import picocli.CommandLine; import tech.tablesaw.api.*; +import tech.tablesaw.columns.strings.AbstractStringColumn; import tech.tablesaw.io.csv.CsvReadOptions; import tech.tablesaw.joining.DataFrameJoiner; import tech.tablesaw.selection.Selection; @@ -44,7 +45,7 @@ requires = {"trips.csv", "persons.csv"}, produces = { "mode_share.csv", "mode_share_per_dist.csv", "mode_users.csv", "trip_stats.csv", - "mode_share_per_%s.csv", "mode_share_per_purpose_per_%s.csv", + "mode_share_per_purpose.csv", "mode_share_per_%s.csv", "population_trip_stats.csv", "trip_purposes_by_hour.csv", "mode_share_distance_distribution.csv", "mode_shift.csv", "mode_chains.csv", "mode_choices.csv", "mode_choice_evaluation.csv", "mode_choice_evaluation_per_mode.csv", @@ -282,11 +283,15 @@ public Integer call() throws Exception { joined.addColumns(dist_group); + TextColumn purpose = joined.textColumn("end_activity_type"); + + // Remove suffix durations like _345 + purpose.set(Selection.withRange(0, purpose.size()), purpose.replaceAll("_[0-9]{2,}$", "")); + writeModeShare(joined, labels); if (groups != null) { groups.writeModeShare(joined, labels, modeOrder, (g) -> output.getPath("mode_share_per_%s.csv", g)); - groups.writeModeSharePerPurpose(joined,modeOrder, (g) -> output.getPath("mode_share_per_purpose_per_%s.csv", g)); } if (persons.containsColumn(ATTR_REF_MODES)) { @@ -310,6 +315,7 @@ public Integer call() throws Exception { tryRun(this::writeTripDistribution, joined); tryRun(this::writeModeShift, joined); tryRun(this::writeModeChains, joined); + tryRun(this::writeModeStatsPerPurpose, joined); return 0; } @@ -508,11 +514,6 @@ private void writeTripPurposes(Table trips) { IntColumn.create("arrival_h", arrival.intStream().toArray()) ); - TextColumn purpose = trips.textColumn("end_activity_type"); - - // Remove suffix durations like _345 - purpose.set(Selection.withRange(0, purpose.size()), purpose.replaceAll("_[0-9]{2,}$", "")); - Table tArrival = trips.summarize("trip_id", count).by("end_activity_type", "arrival_h"); tArrival.column(0).setName("purpose"); @@ -672,6 +673,33 @@ else if (modes.size() > 6) { } } + @SuppressWarnings("unchecked") + private void writeModeStatsPerPurpose(Table trips) { + + Table aggr = trips.summarize("trip_id", count).by("end_activity_type", "main_mode"); + + Comparator cmp = Comparator.comparing(row -> row.getString("end_activity_type")); + aggr = aggr.sortOn(cmp.thenComparing(row -> row.getString("main_mode"))); + + aggr.doubleColumn(aggr.columnCount() - 1).setName("share"); + aggr.column("end_activity_type").setName("purpose"); + + Set purposes = (Set) aggr.column("purpose").asSet(); + + // Norm each purpose to 1 + // It was not clear if the purpose is a string or text colum, therefor this code uses the abstract version + for (String label : purposes) { + DoubleColumn all = aggr.doubleColumn("share"); + Selection sel = ((AbstractStringColumn) aggr.column("purpose")).isEqualTo(label); + + double total = all.where(sel).sum(); + if (total > 0) + all.set(sel, all.divide(total)); + } + + aggr.write().csv(output.getPath("mode_share_per_purpose.csv").toFile()); + } + /** * How shape file filtering should be applied. */ diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java index 9d0ffaaac26..5f7e8894500 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/population/TripByGroupAnalysis.java @@ -147,12 +147,6 @@ void writeModeShare(Table trips, List dists, List modeOrder, Fun } } - void writeModeSharePerPurpose(Table trips, List modeOrder, Function output) { - - // TODO: implement - - } - void groupPersons(Table persons) { for (Group g : groups) { diff --git a/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java b/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java index dca4a209df1..223b0fdaaec 100644 --- a/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java +++ b/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java @@ -76,6 +76,7 @@ void trip() { Assertions.assertThat(out) .isDirectoryContaining("glob:**trip_stats.csv") .isDirectoryContaining("glob:**mode_share.csv") + .isDirectoryContaining("glob:**mode_share_per_purpose.csv") .isDirectoryContaining("glob:**mode_shift.csv"); }