Skip to content

Commit 2437438

Browse files
landonreedlandonreed
committed
fix(validator): add frequency overlap validation; add tests
re #167
1 parent c1b3b23 commit 2437438

File tree

13 files changed

+248
-70
lines changed

13 files changed

+248
-70
lines changed

‎src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public enum NewGTFSErrorType {
1313
ILLEGAL_FIELD_VALUE(Priority.MEDIUM, "Fields may not contain tabs, carriage returns or new lines."),
1414
INTEGER_FORMAT(Priority.MEDIUM, "Incorrect integer format."),
1515
FARE_TRANSFER_MISMATCH(Priority.MEDIUM, "A fare that does not permit transfers has a non-zero transfer duration."),
16+
FREQUENCY_PERIOD_OVERLAP(Priority.MEDIUM, "A frequency for a trip overlaps with another frequency defined for the same trip."),
1617
FLOATING_FORMAT(Priority.MEDIUM, "Incorrect floating point number format."),
1718
COLUMN_NAME_UNSAFE(Priority.HIGH, "Column header contains characters not safe in SQL, it was renamed."),
1819
NUMBER_PARSING(Priority.MEDIUM, "Unable to parse number from value."),

‎src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java

+64-41
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
import com.conveyal.gtfs.model.Calendar;
55
import com.conveyal.gtfs.model.CalendarDate;
66
import com.conveyal.gtfs.model.Entity;
7+
import com.conveyal.gtfs.model.FareAttribute;
8+
import com.conveyal.gtfs.model.Frequency;
79
import com.conveyal.gtfs.model.PatternStop;
810
import com.conveyal.gtfs.model.Route;
911
import com.conveyal.gtfs.model.ScheduleException;
@@ -68,21 +70,21 @@ public interface EntityPopulator<T> {
6870
T populate (ResultSet results, TObjectIntMap<String> columnForName) throws SQLException;
6971

7072
EntityPopulator<Agency> AGENCY = (result, columnForName) -> {
71-
Agency agency = new Agency();
72-
agency.agency_id = getStringIfPresent(result, "agency_id", columnForName);
73-
agency.agency_name = getStringIfPresent(result, "agency_name", columnForName);
74-
agency.agency_url = getUrlIfPresent (result, "agency_url", columnForName);
75-
agency.agency_timezone = getStringIfPresent(result, "agency_timezone", columnForName);
76-
agency.agency_lang = getStringIfPresent(result, "agency_lang", columnForName);
77-
agency.agency_phone = getStringIfPresent(result, "agency_phone", columnForName);
78-
agency.agency_fare_url = getUrlIfPresent (result, "agency_fare_url", columnForName);
79-
agency.agency_email = getStringIfPresent(result, "agency_email", columnForName);
80-
agency.agency_branding_url = getUrlIfPresent (result, "agency_branding_url", columnForName);
73+
Agency agency = new Agency();
74+
agency.agency_id = getStringIfPresent(result, "agency_id", columnForName);
75+
agency.agency_name = getStringIfPresent(result, "agency_name", columnForName);
76+
agency.agency_url = getUrlIfPresent (result, "agency_url", columnForName);
77+
agency.agency_timezone = getStringIfPresent(result, "agency_timezone", columnForName);
78+
agency.agency_lang = getStringIfPresent(result, "agency_lang", columnForName);
79+
agency.agency_phone = getStringIfPresent(result, "agency_phone", columnForName);
80+
agency.agency_fare_url = getUrlIfPresent (result, "agency_fare_url", columnForName);
81+
agency.agency_email = getStringIfPresent(result, "agency_email", columnForName);
82+
agency.agency_branding_url = getUrlIfPresent (result, "agency_branding_url", columnForName);
8183
return agency;
8284
};
8385

8486
EntityPopulator<Calendar> CALENDAR = (result, columnForName) -> {
85-
Calendar calendar = new Calendar();
87+
Calendar calendar = new Calendar();
8688
calendar.service_id = getStringIfPresent(result, "service_id", columnForName);
8789
calendar.start_date = getDateIfPresent (result, "start_date", columnForName);
8890
calendar.end_date = getDateIfPresent (result, "end_date", columnForName);
@@ -104,10 +106,31 @@ public interface EntityPopulator<T> {
104106
return calendarDate;
105107
};
106108

109+
EntityPopulator<FareAttribute> FARE_ATTRIBUTE = (result, columnForName) -> {
110+
FareAttribute fareAttribute = new FareAttribute();
111+
fareAttribute.fare_id = getStringIfPresent(result, "fare_id", columnForName);
112+
fareAttribute.agency_id = getStringIfPresent(result, "agency_id", columnForName);
113+
fareAttribute.price = getDoubleIfPresent(result, "price", columnForName);
114+
fareAttribute.payment_method = getIntIfPresent (result, "payment_method", columnForName);
115+
fareAttribute.transfers = getIntIfPresent (result, "transfers", columnForName);
116+
fareAttribute.transfer_duration = getIntIfPresent (result, "transfer_duration", columnForName);
117+
return fareAttribute;
118+
};
119+
120+
EntityPopulator<Frequency> FREQUENCY = (result, columnForName) -> {
121+
Frequency frequency = new Frequency();
122+
frequency.trip_id = getStringIfPresent(result, "trip_id", columnForName);
123+
frequency.start_time = getIntIfPresent (result, "start_time", columnForName);
124+
frequency.end_time = getIntIfPresent (result, "end_time", columnForName);
125+
frequency.headway_secs = getIntIfPresent (result, "headway_secs", columnForName);
126+
frequency.exact_times = getIntIfPresent (result, "exact_times", columnForName);
127+
return frequency;
128+
};
129+
107130
EntityPopulator<ScheduleException> SCHEDULE_EXCEPTION = (result, columnForName) -> {
108131
ScheduleException scheduleException = new ScheduleException();
109-
scheduleException.name = getStringIfPresent(result, "name", columnForName);
110-
scheduleException.dates = getDateListIfPresent(result, "dates", columnForName);
132+
scheduleException.name = getStringIfPresent (result, "name", columnForName);
133+
scheduleException.dates = getDateListIfPresent (result, "dates", columnForName);
111134
scheduleException.exemplar = exemplarFromInt(getIntIfPresent(result, "exemplar", columnForName));
112135
scheduleException.customSchedule = getStringListIfPresent(result, "custom_schedule", columnForName);
113136
scheduleException.addedService = getStringListIfPresent(result, "added_service", columnForName);
@@ -116,22 +139,22 @@ public interface EntityPopulator<T> {
116139
};
117140

118141
EntityPopulator<Route> ROUTE = (result, columnForName) -> {
119-
Route route = new Route();
120-
route.route_id = getStringIfPresent(result, "route_id", columnForName);
121-
route.agency_id = getStringIfPresent(result, "agency_id", columnForName);
122-
route.route_short_name = getStringIfPresent(result, "route_short_name", columnForName);
123-
route.route_long_name = getStringIfPresent(result, "route_long_name", columnForName);
124-
route.route_desc = getStringIfPresent(result, "route_desc", columnForName);
125-
route.route_type = getIntIfPresent (result, "route_type", columnForName);
126-
route.route_color = getStringIfPresent(result, "route_color", columnForName);
127-
route.route_text_color = getStringIfPresent(result, "route_text_color", columnForName);
128-
route.route_url = getUrlIfPresent (result, "route_url", columnForName);
129-
route.route_branding_url = getUrlIfPresent (result, "route_branding_url", columnForName);
142+
Route route = new Route();
143+
route.route_id = getStringIfPresent(result, "route_id", columnForName);
144+
route.agency_id = getStringIfPresent(result, "agency_id", columnForName);
145+
route.route_short_name = getStringIfPresent(result, "route_short_name", columnForName);
146+
route.route_long_name = getStringIfPresent(result, "route_long_name", columnForName);
147+
route.route_desc = getStringIfPresent(result, "route_desc", columnForName);
148+
route.route_type = getIntIfPresent (result, "route_type", columnForName);
149+
route.route_color = getStringIfPresent(result, "route_color", columnForName);
150+
route.route_text_color = getStringIfPresent(result, "route_text_color", columnForName);
151+
route.route_url = getUrlIfPresent (result, "route_url", columnForName);
152+
route.route_branding_url = getUrlIfPresent (result, "route_branding_url", columnForName);
130153
return route;
131154
};
132155

133156
EntityPopulator<Stop> STOP = (result, columnForName) -> {
134-
Stop stop = new Stop();
157+
Stop stop = new Stop();
135158
stop.stop_id = getStringIfPresent(result, "stop_id", columnForName);
136159
stop.stop_code = getStringIfPresent(result, "stop_code", columnForName);
137160
stop.stop_name = getStringIfPresent(result, "stop_name", columnForName);
@@ -148,7 +171,7 @@ public interface EntityPopulator<T> {
148171
};
149172

150173
EntityPopulator<Trip> TRIP = (result, columnForName) -> {
151-
Trip trip = new Trip();
174+
Trip trip = new Trip();
152175
trip.trip_id = getStringIfPresent(result, "trip_id", columnForName);
153176
trip.route_id = getStringIfPresent(result, "route_id", columnForName);
154177
trip.service_id = getStringIfPresent(result, "service_id", columnForName);
@@ -163,26 +186,26 @@ public interface EntityPopulator<T> {
163186
};
164187

165188
EntityPopulator<ShapePoint> SHAPE_POINT = (result, columnForName) -> {
166-
ShapePoint shapePoint = new ShapePoint();
167-
shapePoint.shape_id = getStringIfPresent(result, "shape_id", columnForName);
168-
shapePoint.shape_pt_lat = getDoubleIfPresent(result, "shape_pt_lat", columnForName);
169-
shapePoint.shape_pt_lon = getDoubleIfPresent(result, "shape_pt_lon", columnForName);
170-
shapePoint.shape_pt_sequence = getIntIfPresent(result, "shape_pt_sequence", columnForName);
189+
ShapePoint shapePoint = new ShapePoint();
190+
shapePoint.shape_id = getStringIfPresent(result, "shape_id", columnForName);
191+
shapePoint.shape_pt_lat = getDoubleIfPresent(result, "shape_pt_lat", columnForName);
192+
shapePoint.shape_pt_lon = getDoubleIfPresent(result, "shape_pt_lon", columnForName);
193+
shapePoint.shape_pt_sequence = getIntIfPresent (result, "shape_pt_sequence", columnForName);
171194
shapePoint.shape_dist_traveled = getDoubleIfPresent(result, "shape_dist_traveled", columnForName);
172195
return shapePoint;
173196
};
174197

175198
EntityPopulator<StopTime> STOP_TIME = (result, columnForName) -> {
176-
StopTime stopTime = new StopTime();
177-
stopTime.trip_id = getStringIfPresent(result, "trip_id", columnForName);
178-
stopTime.arrival_time = getIntIfPresent (result, "arrival_time", columnForName);
179-
stopTime.departure_time = getIntIfPresent (result, "departure_time", columnForName);
180-
stopTime.stop_id = getStringIfPresent(result, "stop_id", columnForName);
181-
stopTime.stop_sequence = getIntIfPresent (result, "stop_sequence", columnForName);
182-
stopTime.stop_headsign = getStringIfPresent(result, "stop_headsign", columnForName);
183-
stopTime.pickup_type = getIntIfPresent (result, "pickup_type", columnForName);
184-
stopTime.drop_off_type = getIntIfPresent (result, "drop_off_type", columnForName);
185-
stopTime.timepoint = getIntIfPresent (result, "timepoint", columnForName);
199+
StopTime stopTime = new StopTime();
200+
stopTime.trip_id = getStringIfPresent(result, "trip_id", columnForName);
201+
stopTime.arrival_time = getIntIfPresent (result, "arrival_time", columnForName);
202+
stopTime.departure_time = getIntIfPresent (result, "departure_time", columnForName);
203+
stopTime.stop_id = getStringIfPresent(result, "stop_id", columnForName);
204+
stopTime.stop_sequence = getIntIfPresent (result, "stop_sequence", columnForName);
205+
stopTime.stop_headsign = getStringIfPresent(result, "stop_headsign", columnForName);
206+
stopTime.pickup_type = getIntIfPresent (result, "pickup_type", columnForName);
207+
stopTime.drop_off_type = getIntIfPresent (result, "drop_off_type", columnForName);
208+
stopTime.timepoint = getIntIfPresent (result, "timepoint", columnForName);
186209
stopTime.shape_dist_traveled = getDoubleIfPresent(result, "shape_dist_traveled", columnForName);
187210
return stopTime;
188211
};

‎src/main/java/com/conveyal/gtfs/loader/Feed.java

+3
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public class Feed {
3636
public final TableReader<Calendar> calendars;
3737
public final TableReader<CalendarDate> calendarDates;
3838
public final TableReader<FareAttribute> fareAttributes;
39+
public final TableReader<Frequency> frequencies;
3940
public final TableReader<Route> routes;
4041
public final TableReader<Stop> stops;
4142
public final TableReader<Trip> trips;
@@ -58,6 +59,7 @@ public Feed (DataSource dataSource, String tablePrefix) {
5859
this.tablePrefix = tablePrefix == null ? "" : tablePrefix;
5960
agencies = new JDBCTableReader(Table.AGENCY, dataSource, tablePrefix, EntityPopulator.AGENCY);
6061
fareAttributes = new JDBCTableReader(Table.FARE_ATTRIBUTES, dataSource, tablePrefix, EntityPopulator.FARE_ATTRIBUTE);
62+
frequencies = new JDBCTableReader(Table.FREQUENCIES, dataSource, tablePrefix, EntityPopulator.FREQUENCY);
6163
calendars = new JDBCTableReader(Table.CALENDAR, dataSource, tablePrefix, EntityPopulator.CALENDAR);
6264
calendarDates = new JDBCTableReader(Table.CALENDAR_DATES, dataSource, tablePrefix, EntityPopulator.CALENDAR_DATE);
6365
routes = new JDBCTableReader(Table.ROUTES, dataSource, tablePrefix, EntityPopulator.ROUTE);
@@ -90,6 +92,7 @@ public ValidationResult validate () {
9092
new MisplacedStopValidator(this, errorStorage, validationResult),
9193
new DuplicateStopsValidator(this, errorStorage),
9294
new FaresValidator(this, errorStorage),
95+
new FrequencyValidator(this, errorStorage),
9396
new TimeZoneValidator(this, errorStorage),
9497
new NewTripTimesValidator(this, errorStorage),
9598
new NamesValidator(this, errorStorage));

‎src/main/java/com/conveyal/gtfs/validator/FrequencyValidator.java

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package com.conveyal.gtfs.validator;
2+
3+
import com.conveyal.gtfs.error.NewGTFSErrorType;
4+
import com.conveyal.gtfs.error.SQLErrorStorage;
5+
import com.conveyal.gtfs.loader.Feed;
6+
import com.conveyal.gtfs.model.Frequency;
7+
import com.conveyal.gtfs.model.Route;
8+
import com.conveyal.gtfs.model.Stop;
9+
import com.conveyal.gtfs.model.StopTime;
10+
import com.conveyal.gtfs.model.Trip;
11+
import com.google.common.collect.ArrayListMultimap;
12+
import com.google.common.collect.HashMultimap;
13+
import com.google.common.collect.ListMultimap;
14+
import com.google.common.collect.Multimap;
15+
16+
import java.util.Collection;
17+
import java.util.List;
18+
19+
public class FrequencyValidator extends FeedValidator {
20+
21+
/**
22+
* Validate frequency entries to ensure that there are no overlapping frequency periods defined for a single trip.
23+
* @param feed
24+
* @param errorStorage
25+
*/
26+
public FrequencyValidator(Feed feed, SQLErrorStorage errorStorage) {
27+
super(feed, errorStorage);
28+
}
29+
30+
private ListMultimap<String, Frequency> frequenciesById = ArrayListMultimap.create();
31+
32+
@Override
33+
public void validate() {
34+
// First, collect all frequencies for each trip ID.
35+
for (Frequency frequency: feed.frequencies) frequenciesById.put(frequency.trip_id, frequency);
36+
// Next iterate over each set of trip-specific frequency periods.
37+
for (String tripId : frequenciesById.keySet()) {
38+
List<Frequency> frequencies = frequenciesById.get(tripId);
39+
if (frequencies.size() < 1) {
40+
// If there are not more than one frequencies defined for the trip, there can be no risk of overlapping
41+
// frequency intervals.
42+
return;
43+
}
44+
// Iterate over each frequency and check its period against the others for overlap.
45+
for (int i = 0; i < frequencies.size(); i++) {
46+
Frequency frequency = frequencies.get(i);
47+
// Iterate over the other frequencies.
48+
for (int j = 0; j < frequencies.size(); j++) {
49+
// No need to check against self.
50+
if (i == j) continue;
51+
Frequency prevFrequency = frequencies.get(j);
52+
if (
53+
// Other frequency wraps current frequency.
54+
frequency.end_time <= prevFrequency.end_time && frequency.start_time >= prevFrequency.start_time ||
55+
// Frequency starts during other frequency.
56+
frequency.start_time >= prevFrequency.start_time && frequency.start_time < prevFrequency.end_time ||
57+
// Frequency ends during other frequency.
58+
frequency.end_time > prevFrequency.start_time && frequency.end_time <= prevFrequency.end_time
59+
) {
60+
registerError(frequency, NewGTFSErrorType.FREQUENCY_PERIOD_OVERLAP);
61+
}
62+
}
63+
}
64+
}
65+
}
66+
}

‎src/main/java/com/conveyal/gtfs/validator/NewTripTimesValidator.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ public void validate () {
6767
String previousTripId = null;
6868
// Order stop times by trip ID and sequence number (i.e. scan through the stops in each trip in order)
6969
for (StopTime stopTime : feed.stopTimes.getAllOrdered()) {
70-
// FIXME all bad references should already be caught elsewhere, this should just be a continue
70+
// All bad references should already be caught elsewhere, this should just be a continue
7171
if (stopTime.trip_id == null) continue;
7272
if (!stopTime.trip_id.equals(previousTripId) && !stopTimesForTrip.isEmpty()) {
7373
processTrip(stopTimesForTrip);

0 commit comments

Comments
 (0)