Skip to content

Commit de82d85

Browse files
abyrdabyrd
committed
import modified code from OpenTripPlanner repo
mainly changes the interface for creating OSM objects and pulling in data from PBF or streams
1 parent b8ecf85 commit de82d85

16 files changed

+155
-151
lines changed

‎.gitignore

+10-14
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,13 @@
1-
*.class
2-
3-
# Mobile Tools for Java (J2ME)
4-
.mtj.tmp/
5-
6-
# Package Files #
7-
*.jar
8-
*.war
9-
*.ear
10-
11-
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
12-
hs_err_pid*
13-
1+
dependency-reduced-pom.xml
2+
target/
3+
build/
4+
*~
145
.classpath
156
.project
167
.settings
17-
target
8+
.gradle
9+
*.ipr
10+
*.iml
11+
*.iws
12+
*.swp
13+
.idea

‎pom.xml

+14
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,20 @@
77
<version>0.1-SNAPSHOT</version>
88
<name>osm-lib</name>
99
<url>http://maven.apache.org</url>
10+
<build>
11+
<plugins>
12+
<plugin>
13+
<groupId>org.apache.maven.plugins</groupId>
14+
<artifactId>maven-compiler-plugin</artifactId>
15+
<version>3.2</version>
16+
<configuration>
17+
<!-- Target Java versions -->
18+
<source>1.7</source>
19+
<target>1.7</target>
20+
</configuration>
21+
</plugin>
22+
</plugins>
23+
</build>
1024
<dependencies>
1125
<dependency>
1226
<groupId>junit</groupId>

‎src/main/java/com/conveyal/osmlib/App.java

-13
This file was deleted.

‎src/main/java/com/conveyal/osmlib/Node.java

+10-11
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ public class Node extends OSMEntity implements Serializable {
88

99
private static final double FIXED_PRECISION_FACTOR = 1e6;
1010

11-
public Node () {
12-
}
11+
public Node () { }
1312

1413
public Node (double lat, double lon) {
1514
setLatLon(lat, lon);
@@ -28,13 +27,13 @@ public void setLatLon (double lat, double lon) {
2827
this.fixedLon = (int)(lon * FIXED_PRECISION_FACTOR);
2928
}
3029

31-
@Override
32-
public Type getType() {
33-
return OSMEntity.Type.WAY;
34-
}
35-
36-
public String toString(){
37-
return "[Node "+getLat()+" "+getLon()+"]";
38-
}
30+
@Override
31+
public Type getType() {
32+
return Type.NODE;
33+
}
3934

40-
}
35+
public String toString() {
36+
return "[Node "+getLat()+" "+getLon()+"]";
37+
}
38+
39+
}

‎src/main/java/com/conveyal/osmlib/NodeGeomFilter.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,4 @@ private boolean inGeom(double lat, double lon) {
5555
return lon > minLon && lon < maxLon && lat > minLat && lon < maxLat;
5656
}
5757

58-
}
58+
}

‎src/main/java/com/conveyal/osmlib/NodeLoader.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,4 @@ public void handleNode(long id, Node node) {
3333
}
3434
};
3535

36-
}
36+
}

‎src/main/java/com/conveyal/osmlib/NodeTracker.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
package com.conveyal.osmlib;
22

3-
import java.util.Map;
4-
3+
import com.google.common.collect.Maps;
54
import org.slf4j.Logger;
65
import org.slf4j.LoggerFactory;
76

8-
import com.google.common.collect.Maps;
7+
import java.util.Map;
98

109
/**
1110
* A sparse bit set capable of handling 64-bit int indexes (like OSM IDs).
@@ -85,4 +84,4 @@ public boolean contains(long x) {
8584
// Finally, load all nodes that are in any of those ways.
8685

8786
// Filters: bbox and tags.
88-
// Use C PBF converter to pre-filter the data. Toolchains.
87+
// Use C PBF converter to pre-filter the data. Toolchains.

‎src/main/java/com/conveyal/osmlib/OSM.java

+59-51
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
package com.conveyal.osmlib;
22

3+
import com.conveyal.osmlib.serializer.NodeSerializer;
4+
import com.conveyal.osmlib.serializer.WaySerializer;
35
import com.google.common.base.Charsets;
46
import com.google.common.primitives.Longs;
57
import com.vividsolutions.jts.geom.Envelope;
68
import org.mapdb.BTreeKeySerializer;
79
import org.mapdb.DB;
810
import org.mapdb.DBMaker;
911
import org.mapdb.Fun.Tuple3;
10-
import com.conveyal.osmlib.serializer.NodeSerializer;
11-
import com.conveyal.osmlib.serializer.WaySerializer;
1212
import org.slf4j.Logger;
1313
import org.slf4j.LoggerFactory;
1414

@@ -21,6 +21,8 @@
2121
/**
2222
* OTP representation of a subset of OpenStreetMap. One or more PBF files can be loaded into this
2323
* object, which serves as a simplistic database for fetching and iterating over OSM elements.
24+
* Using DB TreeMaps is often not any slower than memory. HashMaps are both bigger and slower.
25+
* This is probably because our keys are so small. A hashmap needs to store both the long key and its hash.
2426
*/
2527
public class OSM {
2628

@@ -34,18 +36,23 @@ public class OSM {
3436
/** The MapDB backing this OSM, if any. */
3537
DB db = null;
3638

37-
// Using DB TreeMaps is observed not to be slower than memory. HashMaps are both bigger and slower.
38-
39-
/** If diskPath is null, OSM will be loaded into memory. */
39+
/**
40+
* Construct a new MapDB-based random-access OSM data store.
41+
* If diskPath is null, OSM will be loaded into a temporary file and deleted on shutdown.
42+
* If diskPath is the string "__MEMORY__" the OSM will be stored entirely in memory.
43+
*
44+
* @param diskPath - the file in which to save the data, null for a temp file, or "__MEMORY__" for in-memory.
45+
*/
4046
public OSM (String diskPath) {
4147
DBMaker dbMaker;
4248
if (diskPath == null) {
4349
LOG.info("OSM will be stored in a temporary file.");
44-
dbMaker = DBMaker.newTempFileDB();
50+
dbMaker = DBMaker.newTempFileDB().deleteFilesAfterClose();
4551
} else {
4652
if (diskPath.equals("__MEMORY__")) {
4753
LOG.info("OSM will be stored in memory.");
48-
dbMaker = DBMaker.newMemoryDirectDB(); // off-heap, no garbage collection
54+
// 'direct' means off-heap memory, no garbage collection overhead
55+
dbMaker = DBMaker.newMemoryDirectDB();
4956
} else {
5057
LOG.info("OSM will be stored in file {}.", diskPath);
5158
dbMaker = DBMaker.newFileDB(new File(diskPath));
@@ -57,70 +64,71 @@ public OSM (String diskPath) {
5764
db = dbMaker.asyncWriteEnable()
5865
.transactionDisable()
5966
.compressionEnable()
60-
.cacheSize(100 * 1024 * 1024)
67+
.cacheSize(50 * 1024 * 1024)
6168
.mmapFileEnableIfSupported()
6269
.closeOnJvmShutdown()
6370
.make();
71+
72+
nodes = db.createTreeMap("nodes")
73+
.valueSerializer(new NodeSerializer())
74+
.makeLongMap();
6475

65-
try{
66-
nodes = db.createTreeMap("nodes")
67-
.valueSerializer(new NodeSerializer())
68-
.makeLongMap();
69-
} catch ( IllegalArgumentException ex ){
70-
nodes = db.getTreeMap("nodes");
71-
}
76+
ways = db.createTreeMap("ways")
77+
.valueSerializer(new WaySerializer())
78+
.makeLongMap();
7279

73-
try{
74-
ways = db.createTreeMap("ways")
75-
.valueSerializer(new WaySerializer())
76-
.makeLongMap();
77-
} catch ( IllegalArgumentException ex ){
78-
ways = db.getTreeMap("ways");
79-
}
80-
81-
try{
82-
relations = db.createTreeMap("relations")
83-
.makeLongMap();
84-
} catch ( IllegalArgumentException ex ){
85-
relations = db.getTreeMap("relations");
86-
}
80+
relations = db.createTreeMap("relations")
81+
.makeLongMap();
8782

88-
// Serializer delta compresses the tuple as a whole and packs ints,
83+
// Serializer delta-compresses the tuple as a whole and variable-width packs ints,
8984
// but does not recursively delta-code its elements.
90-
try{
91-
index = db.createTreeSet("spatial_index")
92-
.serializer(BTreeKeySerializer.TUPLE3)
93-
.make();
94-
} catch ( IllegalArgumentException ex ){
95-
index = db.getTreeSet( "spatial_index" );
96-
}
85+
index = db.createTreeSet("spatial_index")
86+
.serializer(BTreeKeySerializer.TUPLE3)
87+
.make();
9788
}
9889

99-
// boolean filterTags
100-
public static OSM fromPBF(String pbfFile) {
101-
LOG.info("Reading entire PBF file '{}'", pbfFile);
102-
Parser parser = new Parser();
103-
parser.parse(pbfFile);
104-
return parser.osm;
90+
// boolean to filter entities on tags, or list of tag keys to retain?
91+
public void loadFromPBFStream (InputStream in) {
92+
LOG.info("Reading PBF stream.");
93+
Parser parser = new Parser(this);
94+
parser.parse(in);
95+
}
96+
97+
public void loadFromPBFFile (String filePath) {
98+
try {
99+
LOG.info("Reading PBF from file '{}'.", filePath);
100+
Parser parser = new Parser(this);
101+
parser.parse(new FileInputStream(filePath));
102+
} catch (FileNotFoundException e) {
103+
LOG.error("Error occurred while parsing PBF file '{}'", filePath);
104+
e.printStackTrace();
105+
}
105106
}
106107

107-
public static OSM fromPBF(String pbfFile, Envelope env) {
108+
// TODO we know that the ordering of entity types within a PBF file is almost always nodes, then ways,
109+
// then relations. Here we are doing three passes over the whole file, but we could get away with one
110+
// and a half by combining the NodeGeomFilter and the WayLoader, then bailing out of the NodeLoader
111+
// as soon as it sees a Way.
112+
// In any case we can't spatially filter PBF data coming from a stream because we'd need to backtrack.
113+
public void loadFromPBF (String pbfFile, Envelope env) {
108114
LOG.info("Reading PBF file '{}' filtering with envelope {}", pbfFile, env);
109-
OSM osm = new OSM("/var/vex/osm");
110115
LOG.info("Finding nodes within the bounding geometry.");
111116
NodeGeomFilter ngf = new NodeGeomFilter(env);
112117
ngf.parse(pbfFile);
118+
LOG.info("LOAD RELATIONS HERE");
113119
LOG.info("Loading ways containing nodes found within the bounding geometry.");
114-
WayLoader wl = new WayLoader(osm, ngf.nodesInGeom);
120+
WayLoader wl = new WayLoader(this, ngf.nodesInGeom);
115121
wl.parse(pbfFile);
116122
LOG.info("Loading nodes used in the retained ways.");
117-
NodeLoader nl = new NodeLoader(osm, wl.nodesInWays);
123+
NodeLoader nl = new NodeLoader(this, wl.nodesInWays);
118124
nl.parse(pbfFile);
119-
LOG.info("Loading relations (which ones?)");
120-
return osm;
121125
}
122126

123-
/** Decode OSM gzipped text format produced by Vanilla Extract. */
127+
/**
128+
* Decode OSM gzipped text format produced by Vanilla Extract.
129+
* It remains to be determined whether this VEX text format is better or worse than the slightly
130+
* more complicated VEX binary format, but it's certainly simpler and cleaner than PBF.
131+
*/
124132
public void loadFromVexStream (InputStream vexStream) throws IOException {
125133
InputStream unzippedStream = new GZIPInputStream(vexStream);
126134
Reader decoded = new InputStreamReader(unzippedStream, Charsets.UTF_8); // UTF8 ENCODING is important
@@ -161,4 +169,4 @@ public void loadFromVexStream (InputStream vexStream) throws IOException {
161169
}
162170
bufferedReader.close();
163171
}
164-
}
172+
}

‎src/main/java/com/conveyal/osmlib/OSMEntity.java

+5-10
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,10 @@ public abstract class OSMEntity implements Serializable {
99

1010
private static final long serialVersionUID = 1L;
1111

12-
// Format: key1=val1;key2=val2
13-
// TODO MapDB can probably serialize these efficiently even if it's a Map<String,String> or a List<Tag>
14-
// Also OSM allows multimap tags using the semicolon:
15-
// http://wiki.openstreetmap.org/wiki/Talk:Semi-colon_value_separator
16-
// Implementing a Set<Pair<String>> with a single string is not necessarily a good idea...
17-
1812
public static enum Type {
1913
NODE, WAY, RELATION;
2014
}
21-
15+
2216
public List<Tag> tags;
2317

2418
public static class Tag implements Serializable {
@@ -80,7 +74,8 @@ public void addTag (String key, String value) {
8074
}
8175
tags.add(new Tag(key, value));
8276
}
83-
84-
public abstract Type getType();
8577

86-
}
78+
/** This feels strange because we're using Enums to duplicate Java type data (Node.class) */
79+
public abstract Type getType();
80+
81+
}

0 commit comments

Comments
 (0)