Add Append Transformation

src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java

@@ -0,0 +1,76 @@
+
+package com.conveyal.datatools.manager.models.transform;
+
+import com.conveyal.datatools.common.status.MonitorableJob;
+import com.conveyal.datatools.manager.models.TableTransformResult;
+import com.conveyal.datatools.manager.models.TransformType;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+public class AppendToFileTransformation extends ZipTransformation {
+
+    public static AppendToFileTransformation create(String csvData, String table) {
+        AppendToFileTransformation transformation = new AppendToFileTransformation();
+        transformation.csvData = csvData;
+        transformation.table = table;
+        return transformation;
+    }
+
+    @Override
+    public void validateParameters(MonitorableJob.Status status) {
+        if (csvData == null) {
+            status.fail("CSV data must not be null");
+        }
+    }
+
+    @Override
+    public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) {
+        String tableName = table + ".txt";
+        Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath());
+        try (
+                FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null);
+                InputStream inputStream = new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8));
+        ) {
+            TransformType type = TransformType.TABLE_MODIFIED;
+
+            Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs);
+
+            final File tempFile = File.createTempFile(tableName + "-temp", ".txt");
+            Files.copy(targetTxtFilePath, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+
+            // Append CSV data into the target file in the temporary copy of file
+            try (OutputStream os = new FileOutputStream(tempFile, true)) {
+                os.write(inputStream.readAllBytes());
+            } catch (Exception e) {
+                status.fail("Failed to write to target file", e);
+            }
+
+            // Copy modified file into zip
+            Files.copy(tempFile.toPath(), targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING);
+
+            final int NEW_LINE_CHARACTER_CODE = 10;
+            int lineCount = (int) csvData.chars().filter(c -> c == NEW_LINE_CHARACTER_CODE).count();
+            zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult(
+                    tableName,
+                    type,
+                    0,
+                    0,
+                    lineCount + 1,
+                    0
+            ));
+        } catch (Exception e) {
+            status.fail("Unknown error encountered while transforming zip file", e);
+        }
+    }
+}

src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java

@@ -32,7 +32,8 @@
     @JsonSubTypes.Type(value = ReplaceFileFromVersionTransformation.class, name = "ReplaceFileFromVersionTransformation"),
     @JsonSubTypes.Type(value = ReplaceFileFromStringTransformation.class, name = "ReplaceFileFromStringTransformation"),
     @JsonSubTypes.Type(value = PreserveCustomFieldsTransformation.class, name = "PreserveCustomFieldsTransformation"),
-    @JsonSubTypes.Type(value = AddCustomFileFromStringTransformation.class, name = "AddCustomFileTransformation")
+    @JsonSubTypes.Type(value = AddCustomFileFromStringTransformation.class, name = "AddCustomFileTransformation"),
+    @JsonSubTypes.Type(value = AppendToFileTransformation.class, name = "AppendToFileTransformation")
 })
 public abstract class FeedTransformation<Target extends FeedTransformTarget> implements Serializable {
     private static final long serialVersionUID = 1L;

src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java

@@ -11,6 +11,7 @@
 import com.conveyal.datatools.manager.models.Snapshot;
 import com.conveyal.datatools.manager.models.TableTransformResult;
 import com.conveyal.datatools.manager.models.transform.AddCustomFileFromStringTransformation;
+import com.conveyal.datatools.manager.models.transform.AppendToFileTransformation;
 import com.conveyal.datatools.manager.models.transform.DeleteRecordsTransformation;
 import com.conveyal.datatools.manager.models.transform.FeedTransformRules;
 import com.conveyal.datatools.manager.models.transform.FeedTransformation;
@@ -192,6 +193,37 @@ void replaceGtfsPlusFileFailsIfSourceIsMissing() throws IOException {
         assertThat(targetVersion.validationResult, Matchers.nullValue());
     }
 
+    @Test
+    void canAppendToStops() throws SQLException, IOException {
+        sourceVersion = createFeedVersion(
+                feedSource,
+                zipFolderFiles("fake-agency-with-only-calendar")
+        );
+        FeedTransformation transformation = AppendToFileTransformation.create(generateStopRow(), "stops");
+        FeedTransformRules transformRules = new FeedTransformRules(transformation);
+        feedSource.transformRules.add(transformRules);
+        Persistence.feedSources.replace(feedSource.id, feedSource);
+        // Create new target version (note: the folder has no stop_attributes.txt file)
+        targetVersion = createFeedVersion(
+                feedSource,
+                zipFolderFiles("fake-agency-with-only-calendar-dates")
+        );
+        LOG.info("Checking assertions.");
+        assertEquals(
+                6, // Magic number should match row count of stops.txt with one extra
+                targetVersion.feedLoadResult.stops.rowCount,
+                "stops.txt row count should equal input csv data # of rows + 1 extra row"
+        );
+        // Check for presence of new stop id in database (one record).
+        assertThatSqlCountQueryYieldsExpectedCount(
+                String.format(
+                        "SELECT count(*) FROM %s.stops WHERE stop_id = '%s'",
+                        targetVersion.namespace,
+                        "new"
+                ),
+                1
+        );
+    }
     @Test
     void canReplaceFeedInfo() throws SQLException, IOException {
         // Generate random UUID for feedId, which gets placed into the csv data.
@@ -282,6 +314,10 @@ private static String generateStopsWithCustomFields() {
             + "\n1234567,customValue3,customValue4";
     }
 
+    private static String generateStopRow() {
+        return "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,";
+    }
+
     private static String generateCustomCsvData() {
         return "custom_column1,custom_column2,custom_column3"
             + "\ncustomValue1,customValue2,customValue3"