fix(snapshots): add normalizeStopTimes flag to preserve stop_sequence vals

re #283

Author: landonreed

src/main/java/com/conveyal/gtfs/GTFS.java

@@ -81,16 +81,24 @@ public static FeedLoadResult load (String filePath, DataSource dataSource) {
      *   1. The tables' id column has been modified to be auto-incrementing.
      *   2. Primary keys may be added to certain columns/tables.
      *   3. Additional editor-specific columns are added to certain tables.
-     * @param feedId        feed ID (schema namespace) to copy from
-     * @param dataSource    JDBC connection to existing database
-     * @return              FIXME should this be a separate SnapshotResult object?
+     * @param feedId                feed ID (schema namespace) to copy from
+     * @param dataSource            JDBC connection to existing database
+     * @param normalizeStopTimes    whether to normalize stop sequence values on snapshot
+     * @return the result of the snapshot
      */
-    public static SnapshotResult makeSnapshot (String feedId, DataSource dataSource) {
-        JdbcGtfsSnapshotter snapshotter = new JdbcGtfsSnapshotter(feedId, dataSource);
+    public static SnapshotResult makeSnapshot (String feedId, DataSource dataSource, boolean normalizeStopTimes) {
+        JdbcGtfsSnapshotter snapshotter = new JdbcGtfsSnapshotter(feedId, dataSource, normalizeStopTimes);
         SnapshotResult result = snapshotter.copyTables();
         return result;
     }
 
+    /**
+     * Overloaded makeSnapshot method that defaults to normalize stop times.
+     */
+    public static SnapshotResult makeSnapshot (String feedId, DataSource dataSource) {
+        return makeSnapshot(feedId, dataSource, true);
+    }
+
     /**
      * Once a feed has been loaded into the database, examine its contents looking for various problems and errors.
      */
@@ -277,7 +285,7 @@ public static void main (String[] args) throws IOException {
             }
             if (namespaceToSnapshot != null) {
                 LOG.info("Snapshotting feed with unique identifier {}", namespaceToSnapshot);
-                FeedLoadResult snapshotResult = makeSnapshot(namespaceToSnapshot, dataSource);
+                FeedLoadResult snapshotResult = makeSnapshot(namespaceToSnapshot, dataSource, false);
                 if (storeResults) {
                     File snapshotResultFile = new File(directory, String.format("%s-snapshot.json", snapshotResult.uniqueIdentifier));
                     LOG.info("Storing validation result at {}", snapshotResultFile.getAbsolutePath());

src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java

@@ -39,6 +39,13 @@ public class JdbcGtfsSnapshotter {
     private static final Logger LOG = LoggerFactory.getLogger(JdbcGtfsSnapshotter.class);
 
     private final DataSource dataSource;
+    /**
+     * Whether to normalize stop_times#stop_sequence values on snapshot (or leave them intact).
+     *
+     * TODO: if more options are added in the future, this should be folded into a SnapshotOptions
+     *   object.
+     */
+    private final boolean normalizeStopTimes;
 
     // These fields will be filled in once feed snapshot begins.
     private Connection connection;
@@ -49,10 +56,13 @@ public class JdbcGtfsSnapshotter {
     /**
      * @param feedId namespace (schema) to snapshot. If null, a blank snapshot will be created.
      * @param dataSource the JDBC data source with database connection details
+     * @param normalizeStopTimes whether to keep stop sequence values intact or normalize to be zero-based and
+     *                           incrementing
      */
-    public JdbcGtfsSnapshotter(String feedId, DataSource dataSource) {
+    public JdbcGtfsSnapshotter(String feedId, DataSource dataSource, boolean normalizeStopTimes) {
         this.feedIdToSnapshot = feedId;
         this.dataSource = dataSource;
+        this.normalizeStopTimes = normalizeStopTimes;
     }
 
     /**
@@ -135,7 +145,7 @@ private TableLoadResult copy (Table table, boolean createIndexes) {
                 // Otherwise, use the createTableFrom method to copy the data from the original.
                 String fromTableName = String.format("%s.%s", feedIdToSnapshot, table.name);
                 LOG.info("Copying table {} to {}", fromTableName, targetTable.name);
-                success = targetTable.createSqlTableFrom(connection, fromTableName);
+                success = targetTable.createSqlTableFrom(connection, fromTableName, normalizeStopTimes);
             }
             // Only create indexes if table creation was successful.
             if (success && createIndexes) {

src/main/java/com/conveyal/gtfs/loader/Table.java

@@ -834,16 +834,19 @@ public void createIndexes(Connection connection, String namespace) throws SQLExc
     /**
      * Creates a SQL table from the table to clone. This uses the SQL syntax "create table x as y" not only copies the
      * table structure, but also the data from the original table. Creating table indexes is not handled by this method.
+     *
+     * Note: the stop_times table is a special case that will optionally normalize the stop_sequence values to be
+     * zero-based and incrementing.
      */
-    public boolean createSqlTableFrom(Connection connection, String tableToClone) {
+    public boolean createSqlTableFrom(Connection connection, String tableToClone, boolean normalizeStopTimes) {
         long startTime = System.currentTimeMillis();
         try {
             Statement statement = connection.createStatement();
             // Drop target table to avoid a conflict.
             String dropSql = String.format("drop table if exists %s", name);
             LOG.info(dropSql);
             statement.execute(dropSql);
-            if (tableToClone.endsWith("stop_times")) {
+            if (tableToClone.endsWith("stop_times") && normalizeStopTimes) {
                 normalizeAndCloneStopTimes(statement, name, tableToClone);
             } else {
                 // Adding the unlogged keyword gives about 12 percent speedup on loading, but is non-standard.