HERE Data SDK for Java & Scala Release 2.35.0

Author: heremaps-bot

README.md

@@ -85,6 +85,6 @@ The following examples demonstrate how to use the Location Library.
 
 ## License
 
-Copyright (C) 2017-2021 HERE Europe B.V.
+Copyright (C) 2017-2022 HERE Europe B.V.
 
 Unless otherwise noted in `LICENSE` files, source code files for specific files or directories, the [LICENSE](LICENSE) in the root applies to all content in this repository.

data-archive/java/avro-example/README.md

@@ -187,7 +187,7 @@ You can use the OLP CLI to create pipeline components and activate the pipeline
 
 ```bash
 olp pipeline create $PIPELINE_NAME --scope $PROJECT_HRN
-olp pipeline template create $PIPELINE_TEMPLATE_NAME stream-4.0 $PATH_TO_JAR com.here.platform.dal.DALMain \
+olp pipeline template create $PIPELINE_TEMPLATE_NAME stream-5.0 $PATH_TO_JAR com.here.platform.dal.DALMain \
     --input-catalog-ids=source --workers=2 --scope $PROJECT_HRN # Note that the value of workers should be greater than or equal to the value of parallelism selected in application.conf
 olp pipeline version create $PIPELINE_VERSION_NAME $PIPELINE_ID $PIPELINE_TEMPLATE_ID \
     "$PATH_TO_CONFIG_FOLDER/pipeline-config.conf" --scope $PROJECT_HRN

data-archive/java/avro-example/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.here.platform.data.archive</groupId>
     <artifactId>data-archive-avro-example</artifactId>
-    <version>0.0.739</version>
+    <version>0.0.746</version>
     <packaging>jar</packaging>
 
     <!-- Meta information section start -->
@@ -23,15 +23,15 @@
         <connection>scm:git:https://github.com/heremaps/here-workspace-examples-java-scala.git</connection>
         <developerConnection>scm:git:[email protected]:heremaps/here-workspace-examples-java-scala.git</developerConnection>
         <url>https://github.com/heremaps/here-workspace-examples-java-scala</url>
-        <tag>2.34.0</tag>
+        <tag>2.35.0</tag>
     </scm>
 
     <!-- Meta information section end -->
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.version>1.8</java.version>
 
-        <sdk-bom.version>2.34.5</sdk-bom.version>
+        <sdk-bom.version>2.35.4</sdk-bom.version>
         <scala.compat.version>2.12</scala.compat.version>
 
         <avro.version>1.10.0</avro.version>
@@ -72,6 +72,7 @@
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
+            <version>${avro.version}</version>
         </dependency>
         <dependency>
             <groupId>com.google.protobuf</groupId>
@@ -193,7 +194,7 @@
                 <groupId>com.here.platform.artifact</groupId>
                 <artifactId>artifact-wagon</artifactId>
                 <version>2.0.5</version>
-                </extension>
+            </extension>
         </extensions>
     </build>
 
@@ -229,6 +230,11 @@
                     <artifactId>flink-streaming-java_${scala.compat.version}</artifactId>
                     <scope>compile</scope>
                 </dependency>
+                <dependency>
+                    <groupId>org.apache.flink</groupId>
+                    <artifactId>flink-clients_${scala.compat.version}</artifactId>
+                    <scope>compile</scope>
+                </dependency>
                 <!-- The following dependencies are added to this profile to address the following-->
                 <!-- There is a known issue where data-archive pom's transitive dependencies are not resolved when running in an IDE-->
                 <dependency>
@@ -249,6 +255,7 @@
                 <dependency>
                     <groupId>org.apache.avro</groupId>
                     <artifactId>avro</artifactId>
+                    <version>${avro.version}</version>
                 </dependency>
                 <dependency>
                     <groupId>com.here.platform.data.client</groupId>

data-archive/java/avro-example/src/main/java/com/here/platform/data/archive/example/AvroHelper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017-2021 HERE Europe B.V.
+ * Copyright (C) 2017-2022 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

data-archive/java/avro-example/src/main/java/com/here/platform/data/archive/example/AvroMultiKeysExample.java

@@ -1,18 +1,14 @@
 /*
- * Copyright (C) 2017-2021 HERE Europe B.V.
- *
+ * Copyright (C) 2017-2022 HERE Europe B.V.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * http://www.apache.org/licenses/LICENSE-2.0
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  * SPDX-License-Identifier: Apache-2.0
  * License-Filename: LICENSE
  */
@@ -128,14 +124,4 @@ public byte[] aggregate(Map<String, Object> keys, Iterator<byte[]> messages) {
     }
     return null;
   }
-
-  @Override
-  public byte[] merge(Map<String, Object> keys, Iterator<byte[]> files) {
-    try {
-      return AvroHelper.mergeAvroFiles(files, SdiiMessage.Message.class);
-    } catch (Exception e) {
-      LOG.error("Merge errors....", e);
-    }
-    return null;
-  }
 }

data-archive/java/avro-example/src/main/java/com/here/platform/data/archive/example/AvroSimpleKeyExample.java

@@ -1,18 +1,14 @@
 /*
- * Copyright (C) 2017-2021 HERE Europe B.V.
- *
+ * Copyright (C) 2017-2022 HERE Europe B.V.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * http://www.apache.org/licenses/LICENSE-2.0
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  * SPDX-License-Identifier: Apache-2.0
  * License-Filename: LICENSE
  */
@@ -133,14 +129,4 @@ public byte[] aggregate(Map<String, Object> keys, Iterator<byte[]> messages) {
     }
     return null;
   }
-
-  @Override
-  public byte[] merge(Map<String, Object> keys, Iterator<byte[]> files) {
-    try {
-      return AvroHelper.mergeAvroFiles(files, SdiiMessage.Message.class);
-    } catch (Exception e) {
-      LOG.error("Merge errors....", e);
-    }
-    return null;
-  }
 }

data-archive/java/avro-example/src/main/resources/application.conf

@@ -87,8 +87,6 @@ aggregation {
 # - A UDF implementation returns an unchecked/runtime exception
 # - A UDF implementation returns null
 # - An indexing attribute value does not pass the Data Archiving Library's validation rules.
-# The error handling strategy you choose does not apply to error scenarios in a User-Defined Function's `merge()` function.
-# This is because the `merge()` function is only used for the Data Archiving Library's internal optimization process of compaction. Any failures in this process will not fail your archiving pipeline.
 error {
   # You can choose one of the following error handling strategies:
   # - "fail" - The pipeline fails on any error from a User-Defined Function's implementation. The pipeline logs the action.

data-archive/java/avro-example/src/test/java/com/here/platform/data/archive/example/AvroExampleRunner.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017-2021 HERE Europe B.V.
+ * Copyright (C) 2017-2022 HERE Europe B.V.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

data-archive/java/avro-example/src/test/java/com/here/platform/data/archive/example/AvroSimpleKeyExampleTest.java

@@ -1,18 +1,14 @@
 /*
- * Copyright (C) 2017-2021 HERE Europe B.V.
- *
+ * Copyright (C) 2017-2022 HERE Europe B.V.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * http://www.apache.org/licenses/LICENSE-2.0
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *
  * SPDX-License-Identifier: Apache-2.0
  * License-Filename: LICENSE
  */
@@ -131,34 +127,6 @@ public void testAggregate() throws IOException {
     assertTrue(aggregateDuration.getCount() > 0);
   }
 
-  @Test
-  public void testMerge() throws IOException {
-    SdiiMessage.Message message1 = prepareSDIIMessage(System.currentTimeMillis(), true, 10d, 10d);
-    SdiiMessage.Message message2 = prepareSDIIMessage(System.currentTimeMillis(), true, 10d, 10d);
-    SdiiMessage.Message message3 = prepareSDIIMessage(System.currentTimeMillis(), true, 10d, 10d);
-    SdiiMessage.Message message4 = prepareSDIIMessage(System.currentTimeMillis(), true, 10d, 10d);
-    example.open(parameters, runtimeContext);
-    byte[] archiveBytesMessage1 =
-        example.aggregate(
-            null, Arrays.asList(message1.toByteArray(), message2.toByteArray()).iterator());
-    byte[] archiveBytesMessage2 =
-        example.aggregate(
-            null, Arrays.asList(message3.toByteArray(), message4.toByteArray()).iterator());
-    File tmpFile = File.createTempFile("test", ".avro");
-    tmpFile.deleteOnExit();
-
-    FileUtils.writeByteArrayToFile(
-        tmpFile,
-        example.merge(
-            new HashMap<>(), Arrays.asList(archiveBytesMessage1, archiveBytesMessage2).iterator()));
-    List<SdiiMessage.Message> list = AvroHelper.fromFile(tmpFile, SdiiMessage.Message.class);
-    assertEquals(4, list.size());
-    assertSDIIMessagesAreEqual(message1, list.get(0));
-    assertSDIIMessagesAreEqual(message2, list.get(1));
-    assertSDIIMessagesAreEqual(message3, list.get(2));
-    assertSDIIMessagesAreEqual(message4, list.get(3));
-  }
-
   private SdiiMessage.Message prepareSDIIMessage(
       long receivedTime, boolean withPathEvent, double longitude, double latitude)
       throws IOException {

data-archive/java/parquet-example/README.md

@@ -186,7 +186,7 @@ You can use the OLP CLI to create pipeline components and activate the pipeline
 
 ```bash
  olp pipeline create $PIPELINE_NAME --scope $PROJECT_HRN
- olp pipeline template create $PIPELINE_TEMPLATE_NAME stream-4.0 $PATH_TO_JAR com.here.platform.dal.DALMain \
+ olp pipeline template create $PIPELINE_TEMPLATE_NAME stream-5.0 $PATH_TO_JAR com.here.platform.dal.DALMain \
       --input-catalog-ids=source --workers=2 --scope $PROJECT_HRN # Note that the value of workers should be greater than or equal to the value of parallelism selected in application.conf
  olp pipeline version create $PIPELINE_VERSION_NAME $PIPELINE_ID $PIPELINE_TEMPLATE_ID \
       "$PATH_TO_CONFIG_FOLDER/pipeline-config.conf" --scope $PROJECT_HRN

data-archive/java/parquet-example/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.here.platform.data.archive</groupId>
     <artifactId>data-archive-parquet-example</artifactId>
-    <version>0.0.739</version>
+    <version>0.0.746</version>
     <packaging>jar</packaging>
 
     <!-- Meta information section start -->
@@ -23,15 +23,15 @@
         <connection>scm:git:https://github.com/heremaps/here-workspace-examples-java-scala.git</connection>
         <developerConnection>scm:git:[email protected]:heremaps/here-workspace-examples-java-scala.git</developerConnection>
         <url>https://github.com/heremaps/here-workspace-examples-java-scala</url>
-        <tag>2.34.0</tag>
+        <tag>2.35.0</tag>
     </scm>
 
     <!-- Meta information section end -->
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.version>1.8</java.version>
 
-        <sdk-bom.version>2.34.5</sdk-bom.version>
+        <sdk-bom.version>2.35.4</sdk-bom.version>
         <scala.compat.version>2.12</scala.compat.version>
 
         <hadoop-client.version>2.7.3</hadoop-client.version>
@@ -198,16 +198,31 @@
                 </property>
             </activation>
             <dependencies>
+                <dependency>
+                    <groupId>org.apache.flink</groupId>
+                    <artifactId>flink-core</artifactId>
+                    <scope>compile</scope>
+                </dependency>
                 <dependency>
                     <groupId>org.apache.flink</groupId>
                     <artifactId>flink-java</artifactId>
                     <scope>compile</scope>
                 </dependency>
+                <dependency>
+                    <groupId>org.apache.flink</groupId>
+                    <artifactId>flink-metrics-core</artifactId>
+                    <scope>compile</scope>
+                </dependency>
                 <dependency>
                     <groupId>org.apache.flink</groupId>
                     <artifactId>flink-streaming-java_${scala.compat.version}</artifactId>
                     <scope>compile</scope>
                 </dependency>
+                <dependency>
+                    <groupId>org.apache.flink</groupId>
+                    <artifactId>flink-clients_${scala.compat.version}</artifactId>
+                    <scope>compile</scope>
+                </dependency>
                 <!-- The following dependencies are added to this profile to address the following-->
                 <!-- There is a known issue where data-archive pom's transitive dependencies are not resolved when running in an IDE-->
                 <dependency>
@@ -228,6 +243,7 @@
                 <dependency>
                     <groupId>org.apache.avro</groupId>
                     <artifactId>avro</artifactId>
+                    <version>1.10.0</version>
                 </dependency>
                 <dependency>
                     <groupId>com.here.platform.data.client</groupId>

data-archive/java/parquet-example/src/main/java/com/here/platform/data/archive/example/CustomProtoSchemaConverter.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (C) 2019-2021 HERE Europe B.V.
+ * Copyright (C) 2019-2022 HERE Europe B.V.
  * SPDX-License-Identifier: Apache-2.0
  *
  * This file is based on commit f84938441be49c665595c936ac631c3e5f171bf9

data-archive/java/parquet-example/src/main/java/com/here/platform/data/archive/example/CustomProtoWriteSupport.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (C) 2019-2021 HERE Europe B.V.
+ * Copyright (C) 2019-2022 HERE Europe B.V.
  * SPDX-License-Identifier: Apache-2.0
  *
  * This file is based on commit f84938441be49c665595c936ac631c3e5f171bf9