From c8c9b0ccda8f003f9f10a725bdd50cca53a3ce76 Mon Sep 17 00:00:00 2001 From: pawelkocinski Date: Mon, 14 Apr 2025 23:43:28 +0200 Subject: [PATCH 1/3] SEDONA-731 add osm nodes parser --- .../osmpbf/extractors/DenseNodeExtractor.java | 2 + .../osmpbf/extractors/NodeExtractor.java | 60 ++++++++++++++++++ .../osmpbf/iterators/BlobIterator.java | 15 ++++- .../src/test/resources/osmpbf/dense.pbf | Bin 0 -> 282 bytes .../src/test/resources/osmpbf/nodes.pbf | Bin 0 -> 242 bytes .../org/apache/sedona/sql/OsmReaderTest.scala | 38 +++++++++++ 6 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/NodeExtractor.java create mode 100644 spark/common/src/test/resources/osmpbf/dense.pbf create mode 100644 spark/common/src/test/resources/osmpbf/nodes.pbf diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/DenseNodeExtractor.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/DenseNodeExtractor.java index f850682be85..0b9ca937488 100644 --- a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/DenseNodeExtractor.java +++ b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/DenseNodeExtractor.java @@ -81,6 +81,8 @@ HashMap parseTags(Osmformat.StringTable stringTable) { keyIndex = keyIndex + 2; } + keyIndex = keyIndex + 1; + return tags; } } diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/NodeExtractor.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/NodeExtractor.java new file mode 100644 index 00000000000..aa7add25cdf --- /dev/null +++ b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/NodeExtractor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sedona.sql.datasources.osmpbf.extractors; + +import java.util.HashMap; +import org.apache.sedona.sql.datasources.osmpbf.build.Osmformat; +import org.apache.sedona.sql.datasources.osmpbf.features.TagsResolver; +import org.apache.sedona.sql.datasources.osmpbf.model.OsmNode; + +public class NodeExtractor { + + Osmformat.PrimitiveGroup primitiveGroup; + Osmformat.PrimitiveBlock primitiveBlock; + + public NodeExtractor( + Osmformat.PrimitiveGroup primitiveGroup, Osmformat.PrimitiveBlock primitiveBlock) { + this.primitiveGroup = primitiveGroup; + this.primitiveBlock = primitiveBlock; + } + + public OsmNode extract(int idx, Osmformat.StringTable stringTable) { + return parse(idx, stringTable); + } + + private OsmNode parse(int idx, Osmformat.StringTable stringTable) { + Osmformat.Node node = primitiveGroup.getNodes(idx); + + long id = node.getId(); + long latitude = node.getLat(); + long longitude = node.getLon(); + + long latOffset = primitiveBlock.getLatOffset(); + long lonOffset = primitiveBlock.getLonOffset(); + long granularity = primitiveBlock.getGranularity(); + + float lat = (float) (.000000001 * (latOffset + (latitude * granularity))); + float lon = (float) (.000000001 * (lonOffset + (longitude * granularity))); + + HashMap tags = + TagsResolver.resolveTags(node.getKeysCount(), node::getKeys, node::getVals, stringTable); + + return new OsmNode(id, lat, lon, tags); + } +} diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/BlobIterator.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/BlobIterator.java index 89a9005771e..ff7a2f88d2d 100644 --- a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/BlobIterator.java +++ b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/BlobIterator.java @@ -27,6 +27,7 @@ import org.apache.sedona.sql.datasources.osmpbf.build.Fileformat.Blob; import org.apache.sedona.sql.datasources.osmpbf.build.Osmformat; import org.apache.sedona.sql.datasources.osmpbf.extractors.DenseNodeExtractor; +import org.apache.sedona.sql.datasources.osmpbf.extractors.NodeExtractor; import org.apache.sedona.sql.datasources.osmpbf.extractors.RelationExtractor; import org.apache.sedona.sql.datasources.osmpbf.extractors.WaysExtractor; import org.apache.sedona.sql.datasources.osmpbf.model.OSMEntity; @@ -68,7 +69,7 @@ public OSMEntity next() { } if (!currentPrimitiveGroup.getNodesList().isEmpty()) { - return null; + return extractNodePrimitiveGroup(); } if (!currentPrimitiveGroup.getWaysList().isEmpty()) { @@ -86,6 +87,18 @@ public OSMEntity next() { return null; } + private OSMEntity extractNodePrimitiveGroup() { + osmEntityIdx += 1; + if (currentPrimitiveGroup.getNodesList().size() == osmEntityIdx) { + nextEntity(); + } + + Osmformat.StringTable stringTable = primitiveBlock.getStringtable(); + + return new NodeExtractor(currentPrimitiveGroup, primitiveBlock) + .extract(osmEntityIdx, stringTable); + } + public OSMEntity extractDenseNodePrimitiveGroup() { if (denseNodesIterator == null) { denseNodesIterator = diff --git a/spark/common/src/test/resources/osmpbf/dense.pbf b/spark/common/src/test/resources/osmpbf/dense.pbf new file mode 100644 index 0000000000000000000000000000000000000000..24082953a99fd93a2d41d1c6b20408e3c5356499 GIT binary patch literal 282 zcmV+#0p^DoW~PR>ZpP1FrD&@)rwa!JiA zPW8)ANiA+-WHZn+vD7mH0MY6SkN^Mx3HMXXcj{1BHuAiZWA+xwwi_i%SwqixTrngu;bFxmYgoGcjq&axpV9G77ObY-!ug z!@wZL%)r2)#LURRpvBC@z~IE!u?}Edx>;F&RlDXj1iM@Tc$`z>^DoW~PR>ZpP1FrD&@*dd7h6(d zc4|>2mykg{4&;vCnY;_Ms_4X5VMXYx2Ox8VvS0PJ=$a3r*4ZvX%Q literal 0 HcmV?d00001 diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/OsmReaderTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/OsmReaderTest.scala index 1764e0dfaf8..d3f9a52c1c1 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/OsmReaderTest.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/OsmReaderTest.scala @@ -25,8 +25,12 @@ import org.testcontainers.containers.MinIOContainer import java.io.FileInputStream +case class Node(id: Long, latitude: Double, longitude: Double, tags: Map[String, String]) + class OsmReaderTest extends TestBaseScala with Matchers { val monacoPath: String = resourceFolder + "osmpbf/monaco-latest.osm.pbf" + val densePath: String = resourceFolder + "osmpbf/dense.pbf" + val nodesPath: String = resourceFolder + "osmpbf/nodes.pbf" import sparkSession.implicits._ @@ -44,6 +48,40 @@ class OsmReaderTest extends TestBaseScala with Matchers { assert(cnt > 0) } + it("should parse normal nodes") { + sparkSession.read + .format("osmpbf") + .load(nodesPath) + .select("id", "location.*", "tags") + .selectExpr( + "id", + "ROUND(latitude, 2) AS latitude", + "ROUND(longitude, 2) AS longitude", + "tags") + .as[Node] + .collect() should contain theSameElementsAs Array( + Node(1002, 48.86, 2.35, Map("amenity" -> "cafe", "name" -> "Cafe de Paris")), + Node(1003, 30.12, 22.23, Map("amenity" -> "bakery", "name" -> "Delicious Pastries")), + Node(1001, 52.52, 13.40, Map("amenity" -> "restaurant", "name" -> "Curry 36"))) + } + + it("should parse dense nodes") { + sparkSession.read + .format("osmpbf") + .load(densePath) + .select("id", "location.*", "tags") + .selectExpr( + "id", + "ROUND(latitude, 2) AS latitude", + "ROUND(longitude, 2) AS longitude", + "tags") + .as[Node] + .collect() should contain theSameElementsAs Array( + Node(1002, 48.86, 2.35, Map("amenity" -> "cafe", "name" -> "Cafe de Paris")), + Node(1003, 30.12, 22.23, Map("amenity" -> "bakery", "name" -> "Delicious Pastries")), + Node(1001, 52.52, 13.40, Map("amenity" -> "restaurant", "name" -> "Curry 36"))) + } + it("should be able to read from osm file on s3") { val container = new MinIOContainer("minio/minio:latest") From 4d9f17a229252a34951b4fcff01bcf25d8ededef Mon Sep 17 00:00:00 2001 From: pawelkocinski Date: Wed, 16 Apr 2025 22:36:52 +0200 Subject: [PATCH 2/3] SEDONA-731 update docs --- docs/tutorial/sql.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/sql.md b/docs/tutorial/sql.md index b835084757e..71e8e7d8a16 100644 --- a/docs/tutorial/sql.md +++ b/docs/tutorial/sql.md @@ -480,7 +480,7 @@ Since v1.7.1, Sedona supports loading OSM PBF file format as a DataFrame. ``` OSM PBF files can contain nodes, ways, and relations. Currently Sedona support -DenseNodes, Ways and Relations. When you load the data you get a DataFrame with the following schema. +Nodes, DenseNodes, Ways and Relations. When you load the data you get a DataFrame with the following schema. ``` root From 8eae00df86518381ec903e759728e320f9c4d22e Mon Sep 17 00:00:00 2001 From: pawelkocinski Date: Fri, 18 Apr 2025 22:11:11 +0200 Subject: [PATCH 3/3] SEDONA-731 update docs --- .../sql/datasources/osmpbf/extractors/NodeExtractor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/NodeExtractor.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/NodeExtractor.java index aa7add25cdf..39e98537cd0 100644 --- a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/NodeExtractor.java +++ b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/NodeExtractor.java @@ -49,6 +49,9 @@ private OsmNode parse(int idx, Osmformat.StringTable stringTable) { long lonOffset = primitiveBlock.getLonOffset(); long granularity = primitiveBlock.getGranularity(); + // https://wiki.openstreetmap.org/wiki/PBF_Format + // latitude = .000000001 * (lat_offset + (granularity * lat)) + // longitude = .000000001 * (lon_offset + (granularity * lon)) float lat = (float) (.000000001 * (latOffset + (latitude * granularity))); float lon = (float) (.000000001 * (lonOffset + (longitude * granularity)));