Shared Stops Support

pom.xml

@@ -268,7 +268,7 @@
             <groupId>com.github.conveyal</groupId>
             <artifactId>gtfs-lib</artifactId>
             <!-- Latest dev build on jitpack.io -->
-            <version>f2ceb59027</version>
+            <version>9837b6499796a0eeb5de76314e6a1f3125d695fb</version>
             <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
                 http://www.slf4j.org/codes.html#multiple_bindings
             -->

src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java

@@ -1292,6 +1292,10 @@ public OtpRunnerManifest createAndUploadManifestAndConfigs(boolean graphAlreadyB
             }
         }
 
+        // upload shared stops file
+        String sharedStopsConfig = this.deployment.parentProject().sharedStopsConfig;
+        if (sharedStopsConfig != null) addStringContentsAsBaseFolderDownload(manifest, "shared_stops.csv", sharedStopsConfig);
+
         // upload otp-runner manifest to s3
         try {
             ObjectMapper mapper = new ObjectMapper();

src/main/java/com/conveyal/datatools/manager/jobs/validation/SharedStopsValidator.java

@@ -0,0 +1,118 @@
+package com.conveyal.datatools.manager.jobs.validation;
+
+import com.conveyal.datatools.common.utils.aws.S3Utils;
+import com.conveyal.datatools.manager.models.Project;
+import com.conveyal.gtfs.error.NewGTFSError;
+import com.conveyal.gtfs.error.NewGTFSErrorType;
+import com.conveyal.gtfs.error.SQLErrorStorage;
+import com.conveyal.gtfs.loader.Feed;
+import com.conveyal.gtfs.model.Stop;
+import com.conveyal.gtfs.validator.FeedValidator;
+import com.csvreader.CsvReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+public class SharedStopsValidator extends FeedValidator {
+    private static final Logger LOG = LoggerFactory.getLogger(SharedStopsValidator.class);
+
+    Feed feed;
+    Project project;
+
+    public SharedStopsValidator(Project project) {
+        super(null, null);
+        this.project = project;
+    }
+
+    // This method can only be run on a SharedStopsValidator that has been set up with a project only
+    public SharedStopsValidator buildSharedStopsValidator(Feed feed, SQLErrorStorage errorStorage) {
+        if (this.project == null) {
+            throw new RuntimeException("Shared stops validator can not be called because no project has been set!");
+        }
+        return new SharedStopsValidator(feed, errorStorage, this.project);
+    }
+    public SharedStopsValidator(Feed feed, SQLErrorStorage errorStorage, Project project) {
+        super(feed, errorStorage);
+        this.feed = feed;
+        this.project = project;
+    }
+
+    @Override
+    public void validate() {
+        String config = project.sharedStopsConfig;
+        if (config == null) {
+            return;
+        }
+
+        CsvReader configReader = CsvReader.parse(config);
+
+        final int STOP_GROUP_ID_INDEX = 0;
+        final int STOP_ID_INDEX = 2;
+        final int IS_PRIMARY_INDEX = 3;
+
+        // Build list of stop Ids.
+        List<String> stopIds = new ArrayList<>();
+        List<Stop> stops = new ArrayList<>();
+
+        for (Stop stop : feed.stops) {
+            stops.add(stop);
+            stopIds.add(stop.stop_id);
+        }
+
+        // Initialize hashmaps to hold duplication info
+        HashMap<String, Set<String>> stopGroupStops = new HashMap<>();
+        HashSet<String> stopGroupsWithPrimaryStops = new HashSet<>();
+
+        try {
+            while (configReader.readRecord()) {
+                String stopGroupId = configReader.get(STOP_GROUP_ID_INDEX);
+                String stopId = configReader.get(STOP_ID_INDEX);
+
+                if (stopId.equals("stop_id")) {
+                    // Swallow header row
+                    continue;
+                }
+
+                stopGroupStops.putIfAbsent(stopGroupId, new HashSet<>());
+                Set<String> seenStopIds = stopGroupStops.get(stopGroupId);
+
+                // Check for SS_01 (stop id appearing in multiple stop groups)
+                if (seenStopIds.contains(stopId)) {
+                    registerError(stops
+                            .stream()
+                            .filter(stop -> stop.stop_id.equals(stopId))
+                            .findFirst()
+                            .orElse(stops.get(0)), NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS
+                    );
+                } else {
+                    seenStopIds.add(stopId);
+                }
+
+                // Check for SS_02 (multiple primary stops per stop group)
+                if (stopGroupsWithPrimaryStops.contains(stopGroupId)) {
+                    registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS, stopGroupId));
+                } else if (configReader.get(IS_PRIMARY_INDEX).equals("true")) {
+                    stopGroupsWithPrimaryStops.add(stopGroupId);
+                }
+
+                // Check for SS_03 (stop_id referenced doesn't exist)
+                // TODO: CHECK FEED ID (adjust the pre-build constructor to include feed_id)
+                if (!stopIds.contains(stopId)) {
+                    registerError(NewGTFSError.forFeed(NewGTFSErrorType.SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST, stopId));
+                }
+            }
+        } catch (IOException e) { LOG.error(e.toString()); }
+        finally {
+            configReader.close();
+        }
+
+
+    }
+}

src/main/java/com/conveyal/datatools/manager/models/Deployment.java

@@ -400,6 +400,16 @@ public void dump (File output, boolean includeManifest, boolean includeOsm, bool
                 out.closeEntry();
             }
         }
+
+        // Include shared_stops.csv, if present
+        if (parentProject().sharedStopsConfig != null) {
+            byte[] sharedStopsConfigAsBytes = parentProject().sharedStopsConfig.getBytes(StandardCharsets.UTF_8);
+            ZipEntry sharedStopsEntry = new ZipEntry("shared_stops.csv");
+            out.putNextEntry(sharedStopsEntry);
+            out.write(sharedStopsConfigAsBytes);
+            out.closeEntry();
+        }
+
         // Finally close the zip output stream. The dump file is now complete.
         out.close();
     }

src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java

@@ -6,6 +6,7 @@
 import com.conveyal.datatools.manager.jobs.ValidateFeedJob;
 import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob;
 import com.conveyal.datatools.manager.jobs.validation.RouteTypeValidatorBuilder;
+import com.conveyal.datatools.manager.jobs.validation.SharedStopsValidator;
 import com.conveyal.datatools.manager.persistence.FeedStore;
 import com.conveyal.datatools.manager.persistence.Persistence;
 import com.conveyal.datatools.manager.utils.HashUtils;
@@ -370,8 +371,14 @@ public void validate(MonitorableJob.Status status) {
                     MTCValidator::new
                 );
             } else {
-                validationResult = GTFS.validate(feedLoadResult.uniqueIdentifier, DataManager.GTFS_DATA_SOURCE,
-                    RouteTypeValidatorBuilder::buildRouteValidator
+                FeedSource fs = Persistence.feedSources.getById(this.feedSourceId);
+                SharedStopsValidator ssv = new SharedStopsValidator(fs.retrieveProject());
+
+                validationResult = GTFS.validate(
+                        feedLoadResult.uniqueIdentifier,
+                        DataManager.GTFS_DATA_SOURCE,
+                        RouteTypeValidatorBuilder::buildRouteValidator,
+                        ssv::buildSharedStopsValidator
                 );
             }
         } catch (Exception e) {
@@ -489,7 +496,9 @@ public boolean hasBlockingIssuesForPublishing() {
             NewGTFSErrorType.SERVICE_WITHOUT_DAYS_OF_WEEK,
             NewGTFSErrorType.TABLE_MISSING_COLUMN_HEADERS,
             NewGTFSErrorType.TABLE_IN_SUBDIRECTORY,
-            NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS
+            NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS,
+            NewGTFSErrorType.MULTIPLE_SHARED_STOPS_GROUPS,
+            NewGTFSErrorType.SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS
         ));
     }
 

src/main/java/com/conveyal/datatools/manager/models/Project.java

@@ -48,6 +48,10 @@ public class Project extends Model {
     /** Last successful auto deploy. **/
     public Date lastAutoDeploy;
 
+    /** CSV formatted shared stops config. **/
+    public String sharedStopsConfig;
+
+
     /**
      * A list of servers that are available to deploy project feeds/OSM to. This includes servers assigned to this
      * project as well as those that belong to no project.